def pr_curve(i):
label = labels[i]
statistics_l = Statistics()
print('Doing label {}'.format(label))
for train_idx, valid_idx in folds:
rng = np.random.RandomState()
rng.seed(seeds[i])
training_fold = developement_df.loc[train_idx, ]
training_fold = training_fold.reset_index(drop=True)
validation_fold = developement_df.loc[valid_idx, ]
validation_fold = validation_fold.reset_index(drop=True)
base_estimators = make_classifiers(method, balanced, labels, random_state=rng)
# Find the best params, then do a final proper calibration.
base_estimator = base_estimators[label]
estimator = RandomizedSearchCV(
estimator=base_estimator, param_distributions=params,
n_iter=60, scoring='f1', cv=3, random_state=rng,
error_score=0.0, n_jobs=1, pre_dispatch='2*n_jobs',
refit=True
)
# Set up the vectorizer for the bag-of-words representation
if vectorizer_method == 'tf-idf':
vectorizer = TfidfVectorizer(
stop_words=['go', '', ' '], binary=binary, lowercase=True,
sublinear_tf=False, max_df=1.0, min_df=0
)
vectorizer.fit(training_fold['terms'].values)
elif vectorizer_method == 'count':
vectorizer = CountVectorizer(
stop_words=['go', '', ' '], binary=binary, lowercase=True
)
vectorizer.fit(training_fold['terms'].values)
# Fit an evaluate the performance of the classifier.
x_train = vectorizer.transform(training_fold['terms'].values)
y_train = np.asarray(training_fold[label].values, dtype=int)
x_valid = vectorizer.transform(validation_fold['terms'].values)
y_valid = np.asarray(validation_fold[label].values, dtype=int)
estimator.fit(x_train, y_train)
for t in thresholds:
y_pred = [int(p[1] >= t) for p in estimator.predict_proba(x_valid)]
precision = precision_score(y_valid, y_pred, labels=[0, 1], pos_label=1)
recall = recall_score(y_valid, y_pred, labels=[0, 1], pos_label=1)
f1 = f1_score(y_valid, y_pred, labels=[0, 1], pos_label=1)
statistics_l.update_statistics(label=t, s_type='Precision', data=precision)
statistics_l.update_statistics(label=t, s_type='Recall', data=recall)
statistics_l.update_statistics(label=t, s_type='F1-Score', data=f1)
statistics_l.frame()['reaction'] = label
return statistics_l
def evaluate_crf_model(x, y, estimator, labels, uniprot=None, verbose=0):
y_pred = np.asarray(estimator.predict(x))
statistics = Statistics()
statistics.update_statistics('all_labels', 'accuracy', estimator.score(x, y))
bin_labels = [0, 1]
for i, l in enumerate(labels):
y_true_binary_l = y[:, i].astype(int)
y_pred_binary_l = y_pred[:, i].astype(int)
label_stats = compute_label_statistics(y_true_binary_l, y_pred_binary_l, labels=bin_labels)
statistics.update_statistics(l, 'Accuracy', accuracy_score(y_true_binary_l, y_pred_binary_l))
statistics.update_statistics(l, 'Specifcity', label_stats[1]['specificity'])
statistics.update_statistics(l, 'Recall', label_stats[1]['sensitivity'])
statistics.update_statistics(l, 'Precision', label_stats[1]['precision'])
statistics.update_statistics(l, 'FDR', label_stats[1]['fdr'])
statistics.update_statistics(l, 'F-Score (beta=0.5)', fbeta_score(
y_true_binary_l, y_pred_binary_l, beta=0.5, labels=bin_labels, average='binary'
))
statistics.update_statistics(l, 'F-Score (beta=1)', fbeta_score(
y_true_binary_l, y_pred_binary_l, beta=1.0, labels=bin_labels, average='binary'
))
try:
roc_auc = roc_auc_score(y_true_binary_l, y_pred_binary_l, average="binary")
statistics.update_statistics(l, 'ROC-AUC', roc_auc)
except (ValueError, AssertionError):
statistics.update_statistics(l, 'ROC-AUC', np.NaN)
try:
pr_auc = average_precision_score(y_true_binary_l, y_pred_binary_l, average="binary")
statistics.update_statistics(l, 'PR-AUC', pr_auc)
except (ValueError, AssertionError):
statistics.update_statistics(l, 'PR-AUC', np.NaN)
if verbose:
for l in labels:
statistics.print_statistics(l)
if uniprot and verbose:
for u, p1, p2 in zip(uniprot, y, y_pred):
print("\t\t\tResult for {} \n\t\t\t\tTrue: \t{} ||| Pred: \t{}".format(u, p1, p2))
return statistics
def multi_label_evaluate(y, y_prob, threshold):
statistics = Statistics()
y_pred = (y_prob >= threshold).astype(int)
y_pred_50 = (y_prob >= 0.5).astype(int)
ranking_loss = label_ranking_loss(y, y_pred)
lraps = label_ranking_average_precision_score(y, y_pred)
ranking_loss_50 = label_ranking_loss(y, y_pred_50)
lraps_50 = label_ranking_average_precision_score(y, y_pred_50)
f1_macro = f1_score(y, y_pred, average='macro')
f1_macro_50 = f1_score(y, y_pred_50, average='macro')
statistics.update_statistics("Multi-Label", "Ranking Loss", ranking_loss)
statistics.update_statistics("Multi-Label", "Ranking Precision", lraps)
statistics.update_statistics("Multi-Label", "Ranking Loss (t=0.5)", ranking_loss_50)
statistics.update_statistics("Multi-Label", "Ranking Precision (t=0.5)", lraps_50)
statistics.update_statistics("Multi-Label", "Macro F1", f1_macro)
statistics.update_statistics("Multi-Label", "Macro F1 (t=0.5)", f1_macro_50)
try:
auc_macro = roc_auc_score(y, y_pred, average='macro')
auc_macro_50 = roc_auc_score(y, y_pred_50, average='macro')
auc_pr_macro = roc_auc_score(y, y_prob, average='macro')
statistics.update_statistics("Multi-Label", "Macro AUC", auc_macro)
statistics.update_statistics("Multi-Label", "Macro AUC (t=0.5)", auc_macro_50)
statistics.update_statistics("Multi-Label", "Macro AUC (Pr)", auc_pr_macro)
except ValueError:
statistics.update_statistics("Multi-Label", "Macro AUC", np.NaN)
statistics.update_statistics("Multi-Label", "Macro AUC (t=0.5)", np.NaN)
statistics.update_statistics("Multi-Label", "Macro AUC (Pr)", np.NaN)
return statistics
def evaluate_model(y, y_prob, label, threshold):
statistics = Statistics()
y_pred = (y_prob[:, 1] >= threshold).astype(int)
bin_labels = [0, 1]
label_stats = compute_label_statistics(y, y_pred, labels=bin_labels)
statistics.update_statistics(label, 'F (beta=1)', f1_score(y, y_pred, average='binary', labels=[0,1], pos_label=1))
statistics.update_statistics(label, 'Specificity', label_stats[1]['specificity'])
statistics.update_statistics(label, 'Recall', label_stats[1]['sensitivity'])
statistics.update_statistics(label, 'Precision', label_stats[1]['precision'])
statistics.update_statistics(label, 'FDR', label_stats[1]['fdr'])
try:
statistics.update_statistics(label, 'ROC-AUC', roc_auc_score(y, y_pred, average='weighted'))
except (ValueError, AssertionError):
statistics.update_statistics(label, 'ROC-AUC', 0.0)
try:
pr_auc = average_precision_score(y, y_pred, average='weighted')
if str(pr_auc) == 'nan':
pr_auc = 0.0
statistics.update_statistics(label, 'PR-AUC', pr_auc)
except (ValueError, AssertionError):
statistics.update_statistics(label, 'PR-AUC', 0.0)
return statistics
