def get_results(ins, oos): #in/out of sample
"""
returns AOROC, AOPR (success), AOPR (failure)
"""
rval = []
y_true = np.hstack((np.ones(len(ins)), np.zeros(len(oos))))
y_score = np.hstack((ins, oos))
rval += [round(roc(y_true, y_score)*100, 2),
round(pr(y_true, y_score)*100, 2)]
y_true = np.hstack((np.zeros(len(ins)), np.ones(len(oos))))
y_score = -y_score
rval += [#round(roc(y_true, y_score)*100, 2),
round(pr(y_true, y_score)*100, 2)]
return rval
def get_AOC(ins, oos): #in/out of sample
rval = []
y_true = np.hstack((np.ones(len(ins)), np.zeros(len(oos))))
y_score = np.vstack((ins, oos))
y_score = y_score.max(axis=1)
rval += [
round(roc(y_true, y_score) * 100, 2),
round(pr(y_true, y_score) * 100, 2)
]
y_true = np.hstack((np.zeros(len(ins)), np.ones(len(oos))))
y_score = -y_score
rval += [ #round(roc(y_true, y_score)*100, 2),
round(pr(y_true, y_score) * 100, 2)
]
return rval
def plot_precision_recall_fscore(self, actural_label, predict_probability, beta=1, plot_ks=True, plot_pr=True, thresholds_density=False, subplot=None):
'''
plot precision recall f-score
Args:
actual_label: real label from test DataFrame
predict_probability: predicted probability like score from model
Returns:
None
'''
beta_show = ('%f' % beta).rstrip('0').rstrip('.')
if subplot is not None:
plt.subplot(subplot)
else:
plt.figure(figsize=self.size)
P, R, thresholds = pr(actural_label, predict_probability)
fpr, tpr, rthresholds = roc_curve(actural_label, predict_probability)
ks = max(abs(tpr - fpr))
thresholds = np.pad(thresholds, (1, 0), mode="constant", constant_values=(0.0, 0.0))
rthresholds = rthresholds[::-1]
dv = np.where(P + R == 0.0, 1.0, beta * beta * P + R)
Fscore = (1 + beta*beta) * P * R / dv
if plot_pr == True:
plt.plot(thresholds, P, label='Precision', color='g')
plt.plot(thresholds, R, label='Recall', color='b')
plt.plot(thresholds, Fscore, label=r'$F_{%s}$' % beta_show, color='r')
if plot_ks == True:
plt.plot(rthresholds, tpr, color='limegreen', label='TPR')
plt.plot(rthresholds, fpr, color='orange', label='FPR')
plt.plot(rthresholds, tpr - fpr, color='skyblue', label='KS = %0.3f' % ks)
plt.title("Score On Threshold", fontsize=self.title_size)
plt.xlabel("Threshold", fontsize=self.fontsize)
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.xticks(self.xticks01, fontsize=self.fontsize)
plt.yticks(self.yticks01, fontsize=self.fontsize)
plt.legend(loc='upper left', fontsize=self.fontsize)
ax2 = plt.gca().twinx()
# ax2.set_ylim(0, 100)
ax2.set_xlim(0, 1)
ax2.grid(False)
if thresholds_density == True:
sns.distplot(rthresholds, bins=500, color='0.75', label="Density", hist=True, kde=False, norm_hist=True, ax=ax2)
else:
sns.distplot(predict_probability, bins=500, color='0.75', label="Density", hist=True, kde=False, norm_hist=True, ax=ax2)
ax2.set_ylabel("Density", fontsize=self.fontsize)
ax2.tick_params(axis="y", labelsize=self.fontsize)
# hist, bins = np.histogram(predict_probability, bins=200, density=True, range=(0, 1))
# width = np.diff(bins)
# center = (bins[:-1] + bins[1:]) / 2
# plt.bar(center, hist, align='center', width=width, facecolor='0.75', alpha=0.5, label="Density")
# plt.hist(predict_probability, bins=256, normed=True, facecolor='0.75', alpha=0.5)
ax2.legend(fontsize=self.fontsize)
if subplot is None:
plt.tight_layout()
self.show()
def plot_PR(self, actural_label, predict_probability, beta=1, subplot=None):
'''
展示P,R的结果，并画出P-R图
Args:
actural_label: 真实结果
predict_probability: 预测结果的概率
'''
beta_show = ('%f' % beta).rstrip('0').rstrip('.')
if subplot is not None:
plt.subplot(subplot)
else:
plt.figure(figsize=self.size)
P, R, thresholds = pr(actural_label, predict_probability)
dv = np.where(P + R == 0.0, 1.0, beta * beta * P + R)
Fscore = (1 + beta*beta) * P * R / dv
plt.plot(R, P, color='g', label='Precision')
plt.plot(R, Fscore, color='r', label=r'$F_{%s}$' % beta_show)
plt.title("PR curve", fontsize=self.title_size)
plt.xlabel("Recall", fontsize=self.fontsize)
plt.legend(loc='upper right', fontsize=self.fontsize)
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.xticks(self.xticks01, fontsize=self.fontsize)
plt.yticks(self.yticks01, fontsize=self.fontsize)
if subplot is None:
plt.tight_layout()
self.show()
def sensiment_analyzer(tweet_input,
tweet_output,
delimiter=',',
tweet_index=3):
true_moods = []
estimated_moods = [] # counting frequency of tweet
mood_mapping = {0: -1, 4: 1}
with open(tweet_input, "r") as tweetfile:
rd = csv.reader(tweetfile)
for row in rd:
try:
s = unicode()
except UnicodeDecodeError:
s = str(s).encode('string_escape')
s = unicode(s)
s = row[tweet_index]
if not mood_mapping.has_key(int(row[0])):
continue
res = SentiStrength(s)
mood = int(res.split()[2])
estimated_moods.append(mood)
true_mood = mood_mapping[int(row[0])]
true_moods.append(true_mood)
print true_mood, '\t', mood
bPrecis, bRecall, bFscore, bSupport = pr(true_moods,
estimated_moods,
average='macro',
labels=[-1, 1])
print bPrecis, bRecall, bFscore, bSupport
def evaluate(model,
iterator_function,
_batch_count,
cuda_device,
output_buffer=sys.stderr):
if output_buffer is not None:
print(_batch_count, file=output_buffer)
model.eval()
with torch.no_grad():
predictions = []
expectations = []
batch_generator = range(_batch_count)
if output_buffer is not None:
batch_generator = tqdm(batch_generator)
for _ in batch_generator:
features, targets = iterator_function()
if cuda_device != -1:
features = features.cuda(device=cuda_device)
probs, _, _ = model(example_batch=features)
batch_pred = np.argmax(probs.detach().cpu().numpy(),
axis=-1).tolist()
batch_tgt = targets.detach().cpu().numpy().tolist()
predictions.extend(batch_pred)
expectations.extend(batch_tgt)
model.train()
return acc(expectations, predictions) * 100, \
pr(expectations, predictions) * 100, \
rc(expectations, predictions) * 100, \
f1(expectations, predictions) * 100,
def SVMEvaluation(self, y_dev, X_dev):
# obtenemos precision, recall y f1 comparando el gold standard (y_dev) con las predicciones
predicted = self.model.predict(X_dev)
bPrecis, bRecall, bFscore, bSupport = pr(y_dev, predicted, average='macro')
# mostramos resultados
bAcuracy = ac(y_dev, predicted)
print(classification_report(y_dev, predicted))
print(bAcuracy,bPrecis,bRecall,bFscore)
def build_classifier_and_test(train_X,
train_y,
test_X,
test_y,
clf,
print_train_result=True):
clf.fit(train_X, train_y)
if print_train_result == True:
p_tr = clf.predict(train_X)
print("Train Accuracy:\t", acc(train_y, p_tr))
print("Train Precision:\t", pr(train_y, p_tr))
print("Train Recall_score:\t", rc(train_y, p_tr))
print("Train F-score:\t", f1(train_y, p_tr))
predicted = clf.predict(test_X)
print("Accuracy:\t", acc(test_y, predicted))
print("Precision:\t", pr(test_y, predicted))
print("Recall_score:\t", rc(test_y, predicted))
print("F-score:\t", f1(test_y, predicted))
def get_statistics(json_per_document, concat_targets, concat_predictions):
from sklearn.metrics import precision_recall_fscore_support as pr
prec, rec, f1, _ = pr(
[wnut_mappings[p.lower()] for p in concat_predictions], [
"group" if t.lower() is "corporation" else t.lower()
for t in concat_targets
],
labels=wnut_labels)
return {"precision": prec, "recall": rec, "f1": f1}
def clone_analysis(data_paths):
code = []
labels = []
positives = 0
for file_name in data_paths:
data = json.load(open(file_name))
for example in data:
code.append(example['tokenized'])
l = 0
if 'label' in example.keys():
l = int(example['label'])
elif 'lebel' in example.keys():
l = int(example['lebel'])
elif 'leble' in example.keys():
l = int(example['leble'])
elif 'lable' in example.keys():
l = int(example['lable'])
if l > 1:
l = 1
positives += l
labels.append(l)
print(len(code), len(labels), positives, len(labels) - positives)
vectorizer = TfidfVectorizer(input=code,
lowercase=False,
ngram_range=(1, 3))
X = vectorizer.fit_transform(code)
model = KMeans(n_clusters=10, max_iter=100)
model.fit(X)
y = model.predict(X)
cluster_to_positive = [0] * 10
cluster_to_negative = [0] * 10
for pred, label in zip(y, labels):
if label == 1:
cluster_to_positive[pred] += 1
else:
cluster_to_negative[pred] += 1
print(cluster_to_positive)
print(cluster_to_negative)
percentages = [
float(p) / (p + n)
for p, n in zip(cluster_to_positive, cluster_to_negative)
]
for p in percentages:
print(p)
for _ in range(5):
XTrain, XTest, YTrain, YTest = train_test_split(X,
labels,
test_size=0.2)
model = RandomForestClassifier()
model.fit(XTrain, YTrain)
predicted = model.predict(XTest)
print('%.3f\t%.3f\t%.3f\t%.3f' %
(acc(YTest, predicted) * 100, pr(YTest, predicted) * 100,
rc(YTest, predicted) * 100, f1(YTest, predicted) * 100))
pass
def get_AOC(ins, oos): #in/out of sample
# TODO: order of the last 2???
"""
returns AOROC, AOPR (success), AOPR (failure)
"""
rval = []
y_true = np.hstack((np.ones(len(ins)), np.zeros(len(oos))))
y_score = np.vstack((ins, oos))
# TODO: use different scores (e.g. entropy, acq fns)
y_score = y_score.max(axis=1)
#print y_score
#import ipdb; ipdb.set_trace()
rval += [
round(roc(y_true, y_score) * 100, 2),
round(pr(y_true, y_score) * 100, 2)
]
y_true = np.hstack((np.zeros(len(ins)), np.ones(len(oos))))
y_score = -y_score
rval += [ #round(roc(y_true, y_score)*100, 2),
round(pr(y_true, y_score) * 100, 2)
]
return rval
def result(original, test):
bPrecis, bRecall, bFscore, bSupport = pr(original, test, average='binary')
count_tp = 0
count_tn = 0
count_fp = 0
count_fn = 0
for ind, value in enumerate(test):
if value == original[ind]: #true
if value: #true, so tp
count_tp = count_tp + 1
else:
count_tn = count_tn + 1
else:
if value:
count_fp = count_fp + 1
else:
count_fn = count_fn + 1
def run(synonyms,
depth,
threshold,
combine,
dataset="dev",
blacklist=["in", "of", "on"]):
print("Initializng modules...")
text_to_4lang = TextTo4lang(lang="en")
data = read_sherliic("data/" + dataset + ".csv",
ud_path="data/relation_index.tsv",
keep_context=True)
data_frame = build_graph(data)
data['premise_text'] = data["prem_argleft"] + " " + \
data["premise"] + " " + data["prem_argright"]
data['hyp_text'] = data["hypo_argleft"] + " " + \
data["hypothesis"] + " " + data["hypo_argright"]
preds = process(text_to_4lang, data_frame, synonyms, depth, threshold,
combine, blacklist)
bPrecis, bRecall, bFscore, bSupport = pr(data_frame.score.tolist(), preds)
print("Precision: " + str(bPrecis[1]))
print("Recall: " + str(bRecall[1]))
print("Fscore: " + str(bFscore[1]))
tn, fp, fn, tp = cm(data_frame.score.tolist(), preds).ravel()
print("Scores")
print("TN: " + str(tn))
print("FP: " + str(fp))
print("FN: " + str(fn))
print("TP: " + str(tp))
with open("sherlic_output.txt", "w+") as f:
for i, pred in enumerate(preds):
premise = data.premise_text[i]
hypothesis = data.hyp_text[i]
f.write(
str(premise) + " " + str(hypothesis) + " " + str(pred) + "\n")
word_features = get_word_features(get_words_in_tweets(train_data, stop_data))
training_set = nltk.classify.apply_features(extract_features,train_data)
classifier = nltk.NaiveBayesClassifier.train(training_set);
print 'classifier defined'
predicted = []
observed = []
for items in test_data:
sentence = ' '.join(items[0])
predicted.append(classifier.classify(extract_features(sentence.split())))
observed.append(items[1])
#print word_features
precision, recall, fscore, support = pr(observed, predicted)
print('precision: {}'.format(precision))
print('recall: {}'.format(recall))
print('fscore: {}'.format(fscore))
print ('Most Informative Features :')
classifier.show_most_informative_features(5)
print(classification_report(observed, predicted, target_names=['negative','neutral','positive','irrelevant']))
print 'Confusion Matrix'
print nltk.ConfusionMatrix( observed, predicted )
#fill out frequecy distributions, incrementing the counter of each word
def run(synonyms,
filtering,
depth,
threshold,
language,
data_type,
votes,
blacklist,
port,
combine,
wordnet_only=False):
print("Initializng modules...")
graded = True if data_type == "graded" else False
data_frame = read(language, graded=graded)
supported_languages = ["en", "it", "de"]
if language not in supported_languages:
raise Exception("Not supported language")
text_to_4lang = TextTo4lang(lang=language, port=port)
if not wordnet_only:
fourlang_votes = process_fourlang_votes(text_to_4lang, language,
data_frame, synonyms,
filtering, depth, threshold,
blacklist, combine)
else:
fourlang_votes = len(data_frame) * [0]
if votes:
if language == "it" or language == "en":
preds = process(language, data_frame, fourlang_votes)
else:
preds = process_de(data_frame, fourlang_votes)
else:
preds = fourlang_votes
bPrecis, bRecall, bFscore, bSupport = pr(data_frame.score.tolist(),
fourlang_votes)
print("4lang")
print("Precision: " + str(bPrecis[1]))
print("Recall: " + str(bRecall[1]))
print("Fscore: " + str(bFscore[1]))
tn, fp, fn, tp = cm(data_frame.score.tolist(), fourlang_votes).ravel()
print("Scores")
print("TN: " + str(tn))
print("FP: " + str(fp))
print("FN: " + str(fn))
print("TP: " + str(tp))
bPrecis, bRecall, bFscore, bSupport = pr(data_frame.score.tolist(), preds)
print("Voting")
print("Precision: " + str(bPrecis[1]))
print("Recall: " + str(bRecall[1]))
print("Fscore: " + str(bFscore[1]))
tn, fp, fn, tp = cm(data_frame.score.tolist(), preds).ravel()
print("Scores")
print("TN: " + str(tn))
print("FP: " + str(fp))
print("FN: " + str(fn))
print("TP: " + str(tp))
with open("semeval_output.txt", "w+") as f:
for i, pred in enumerate(preds):
premise = data_frame.premise[i]
hypothesis = data_frame.hypothesis[i]
f.write(
str(premise) + " " + str(hypothesis) + " " + str(pred) + "\n")
def calculate_f1(gold, pred):
bPrecis, bRecall, bFscore, bSupport = pr(gold, pred, average='binary')
print(bFscore, bRecall, bFscore)
},
{ # 'csv_file': 'data/test_bin_classify_predict_result2.csv',
'csv_file':
'/disk1/home/xiaj/dev/FlaskFace/predicts_retinaface_inceptionresnetv1.csv',
'name': 'our'
}
]
plt.figure()
for pfile in pred_file:
preds = tools.load_csv(pfile['csv_file'])
preds = preds.astype(np.float32)
y_true, y_score = preds[:, 0], preds[:, 1:]
pos_score = y_score[:, 1]
P, R, thresholds = pr(y_true, pos_score)
n_classes = len(set(y_true))
plt.plot(R,
P,
label='{}'.format(pfile['name']),
color='deeppink',
linestyle='-',
linewidth=2)
lw = 2
plt.plot([0, 1], [0, 1], 'k--', lw=lw)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('Recall')
plt.ylabel('Precision')
else:
print("should be false:")
print(description)
print(original)
count = count + 1
else:
if original:
print("should be true:")
print(description)
print(original)
count = count + 1
line_count += 1
print("RESULT COUNT")
print(count)
bPrecis, bRecall, bFscore, bSupport = pr(alternate_original,
alternate_test,
average='binary')
print("accuracy", accuracy_score(alternate_original, alternate_test))
print(bPrecis)
print(bRecall)
print(bFscore)
# descriptions_api = {}
all_descriptions = []
negative_batch=x_n)
repr = representation.detach().cpu().numpy()
prediction_classes = np.argmax(prediction_prob.detach().cpu().numpy(),
axis=-1)
# print(
# "Epoch %3d, Loss: %10.4f, Accuracy: %5.2f, Precision: %5.2f, Recall: %5.2f, F1: %5.2f" % (
# epoch, batch_loss.detach().cpu().item(),
# acc(targets, prediction_classes), pr(targets, prediction_classes),
# rc(targets, prediction_classes), f1(targets, prediction_classes)
# )
# )
if epoch % 1 == 0:
prediction_prob, representation, batch_loss = model(
example_batch=test_x, targets=test_y)
repr = representation.detach().cpu().numpy()
prediction_classes = np.argmax(
prediction_prob.detach().cpu().numpy(), axis=-1)
print('=' * 100)
print(
"Test %3d, Loss: %10.4f, Accuracy: %5.2f, Precision: %5.2f, Recall: %5.2f, F1: %5.2f"
% (epoch, batch_loss.detach().cpu().item(),
acc(test_y,
prediction_classes), pr(test_y, prediction_classes),
rc(test_y,
prediction_classes), f1(test_y, prediction_classes)))
print('=' * 100)
plot_embedding(repr, test_y, title='Epoch %d' % epoch)
batch_loss.backward()
optimizer.step()
pass
if not line.startswith('#'):
l = line.split("\t")
if not l[0] == '\n':
s = str(l[1])
s = s[0:len(s)-1]
ans.append(s)
for line in fO:
if not line.startswith('#'):
l = line.split("\t")
if not l[0] == '\n':
s = str(l[1])
s = s[0:len(s)-1]
out.append(s)
label=pd.Series(ans).unique()
print "\nlabels:" + str(label)
prf = pr(ans,out,labels=label,beta=1,average='weighted')
print "\nPresicion:" + str(prf[0])
print "\nRecall:" + str(prf[1])
print "\nF Score:" + str(prf[2])
acp = acc(ans,out,True)
act = acc(ans,out,False)
acp = acp*100
print "\nAccuracy:"+str(acp)+"% "+str(act)+"/"+str(len(ans))
report = cr(ans,out,label)
print str(report)
prf = pr(ans,out,labels=label,beta=1,average=None)
sc = pd.DataFrame(index=['Precision','Recall','F Score','Support'],columns=label)
sc[:]=prf[:]
sc=pd.DataFrame.transpose(sc)
print "\n\n"
print str(sc)
test_data = []
#training corpus has the elements: Topic/sentiment/tweet id/tweet date/tweet text
#tweets are shuffed before partitioining into training and test data
for lines in train_tweets:
words = ' '.join(lines[0])
sentiment = lines[1]
words_filtered = [e.lower() for e in words.split() if len(e) >= 3]
data.append((words_filtered, sentiment))
word_features = get_word_features(get_words_in_tweets(data))
training_set = nltk.classify.apply_features(extract_features, data)
classifier = nltk.NaiveBayesClassifier.train(training_set)
predicted = []
observed = []
for items in test_tweets:
tweets = items[0]
sentence = ' '.join(tweets)
predicted.append(classifier.classify(extract_features(sentence.split())))
observed.append(items[1])
#print word_features
precision, recall, fscore, support = pr(observed, predicted)
print('precision: {}'.format(precision))
print('recall: {}'.format(recall))
print('fscore: {}'.format(fscore))
print('support: {}'.format(support))
# 26.2
end = time.time()
end - start
# 27. Plot learning curve
print('Plot metrics during training...')
ax = lgb.plot_metric(evals_result, metric='binary_logloss')
plt.show()
# 28. Plot precision/Recall curve
# Ref: https://scikit-learn.org/stable/auto_examples/model_selection/plot_precision_recall.html#sphx-glr-auto-examples-model-selection-plot-precision-recall-py
from sklearn.metrics import precision_recall_curve as pr
y_pred = model.predict(X_test)
precision,recall,_ = pr(y_test,y_pred) # Reruns a tuple of three arrays
# precision, recall, thresholds
plt.plot(precision,recall)
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.show()
######## FEATURE IMPORTANCE ###########
# 29
# Column wise imporatnce. Default Criteria: "split".
# "split": Result contains numbers of times feature is used in a model.
# “gain”: Result contains total information-gains of splits
# which use the feature
from sklearn.metrics import recall_score as rec
data = pd.read_csv("transfusion.csv", delimiter=',')
#variable selection, splitting into features and class
features = data.drop(["Donated"], axis=1)
clas = data["Donated"]
#splitting into training and testing
f_train, f_test, c_train, c_test = tr(features,
clas,
test_size=0.2,
random_state=1)
print("Features Training\n", f_train)
print("Features Testing\n", f_test)
print("Class Training\n", c_train)
print("Class Testing\n", c_test)
naive = GaussianNB()
naive.fit(f_train, c_train)
hasil = naive.predict(f_test)
print("hasil", hasil)
print("f1", f1(c_test, hasil, average='macro'))
print("acc", acc(c_test, hasil))
print("precision", pr(c_test, hasil, average='macro'))
print("recall", rec(c_test, hasil, average='macro'))
