def get_scores(y_true,y_pred):
brier_score = brier_score_loss(y_true,y_pred)
log_score = log_loss(y_true,y_pred)
roc_score = roc_auc_score(y_true, y_pred)
pr_score = average_precision_score(y_true,y_pred)
r2score = r2_score(y_true,y_pred)
return math.sqrt(brier_score),log_score,roc_score,pr_score,r2score
def auc_metrics(self, y, ypreds):
self.log("Features Used: {}".format(self.featurizer.get_feature_names()))
auc = roc_auc_score(y, ypreds)
self.log("\tROC AUC: {}".format(auc))
prc = average_precision_score(y, ypreds, average="weighted")
self.log("\tPrecision-Recall AUC: {}".format(prc))
return auc, prc
def get_scores(shots):
y_true = [shot.result for shot in shots]
y_pred = [shot.pred for shot in shots]
brier_score = brier_score_loss(y_true,y_pred)
log_score = log_loss(y_true,y_pred)
roc_score = roc_auc_score(y_true, y_pred)
pr_score = average_precision_score(y_true,y_pred)
r2score = r2_score(y_true,y_pred)
return math.sqrt(brier_score),log_score,roc_score,pr_score,r2score
def plot_precision_recall(y_true, y_pred):
from matplotlib import pyplot as plt
if platform == 'linux':
plt.switch_backend('agg')
precition, recall, _ = precision_recall_curve(y_true, y_pred)
score = average_precision_score(y_true, y_pred)
plt.plot(precition,
recall,
label='curva. Area: {:.3f}'.format(score),
lw=1)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.legend()
plt.savefig('prec_rec.png')
def run_svm_decision_distance(test, train, agreement=1):
"""
:param test:
:param train:
:param agreement:
:return:
"""
from sklearn.pipeline import Pipeline
svc = Pipeline([("svm",
SVC(class_weight='balanced',
kernel='rbf',
C=0.7,
gamma=0.001,
random_state=0))])
features = get_experimential_pipeline(train)
X_train = features.fit_transform(train)
y = [1 if sent.label >= agreement else 0 for sent in train]
X_train, y = balance(X_train, y)
print("Start training SVM.")
svc.fit(X_train, y)
print("Finished training SVM.")
X = features.fit_transform(test)
y_pred_proba = svc.decision_function(X)
y_pred_proba = MinMaxScaler().fit_transform(y_pred_proba).tolist()
y_pred = svc.predict(X)
for sent, prob, pred_label in zip(test, y_pred_proba, y_pred):
sent.pred = prob
sent.pred_label = pred_label
y_true = [1 if s.label >= agreement else 0 for s in test]
print(average_precision_score(y_true, y_pred_proba))
return test
def run_svm_prob(test, train, agreement=1, C=1, gamma=0.0001):
"""
:param test:
:param train:
:param agreement:
:return:
"""
svc = SVC(class_weight='balanced',
kernel='rbf',
C=C,
gamma=gamma,
probability=True,
random_state=0)
features = get_experimential_pipeline(train)
X_train = features.fit_transform(train)
y = [1 if sent.label >= agreement else 0 for sent in train]
print("Start training SVM.")
svc.fit(X_train, y)
print("Finished training SVM.")
X = features.fit_transform(test)
y_pred_proba = svc.predict_proba(X)
y_pred_proba = MinMaxScaler().fit_transform(
[pred[1] for pred in y_pred_proba]).tolist()
y_pred = svc.predict(X)
for sent, prob, pred_label in zip(test, y_pred_proba, y_pred):
sent.pred = prob
sent.pred_label = pred_label
y_true = [1 if s.label >= agreement else 0 for s in test]
print(average_precision_score(y_true, y_pred_proba))
return test
