def evaluate_model(y, y_prob, label, threshold):
statistics = Statistics()
y_pred = (y_prob[:, 1] >= threshold).astype(int)
bin_labels = [0, 1]
label_stats = compute_label_statistics(y, y_pred, labels=bin_labels)
statistics.update_statistics(label, 'F (beta=1)', f1_score(y, y_pred, average='binary', labels=[0,1], pos_label=1))
statistics.update_statistics(label, 'Specificity', label_stats[1]['specificity'])
statistics.update_statistics(label, 'Recall', label_stats[1]['sensitivity'])
statistics.update_statistics(label, 'Precision', label_stats[1]['precision'])
statistics.update_statistics(label, 'FDR', label_stats[1]['fdr'])
try:
statistics.update_statistics(label, 'ROC-AUC', roc_auc_score(y, y_pred, average='weighted'))
except (ValueError, AssertionError):
statistics.update_statistics(label, 'ROC-AUC', 0.0)
try:
pr_auc = average_precision_score(y, y_pred, average='weighted')
if str(pr_auc) == 'nan':
pr_auc = 0.0
statistics.update_statistics(label, 'PR-AUC', pr_auc)
except (ValueError, AssertionError):
statistics.update_statistics(label, 'PR-AUC', 0.0)
return statistics
def evaluate_model(y, y_pred, y_pred_prob, label, statistics, uniprot=None, verbose=0):
y_pred_prob_1 = [x[1] for x in y_pred_prob]
if uniprot:
for u, p1, p2 in zip(uniprot, y, y_pred_prob_1):
print("\t\t\tResult for {}, {} \n\t\t\t\tTrue: \t{} ||| Pred: \t{}".format(label, u, p1, p2))
label_stats = compute_label_statistics(y, y_pred, labels=labels)
statistics.update_statistics(label, 'Accuracy', accuracy_score(y, y_pred))
statistics.update_statistics(label, 'F (beta=0.5)', fbeta_score(y, y_pred, beta=0.5, labels=[0, 1], average='binary'))
statistics.update_statistics(label, 'F (beta=1)', fbeta_score(y, y_pred, beta=1.0, labels=[0, 1], average='binary'))
statistics.update_statistics(label, 'Specificity', label_stats[1]['specificity'])
statistics.update_statistics(label, 'Recall', label_stats[1]['sensitivity'])
statistics.update_statistics(label, 'Precision', label_stats[1]['precision'])
statistics.update_statistics(label, 'FDR', label_stats[1]['fdr'])
try:
statistics.update_statistics(label, 'ROC-AUC', roc_auc_score(y, y_pred, average="weighted"))
except (ValueError, AssertionError):
statistics.update_statistics(label, 'AUC', 0.0)
try:
pr_auc = average_precision_score(y, y_pred, average="weighted")
if str(pr_auc) == 'nan':
pr_auc = 0.0
statistics.update_statistics(label, 'PR-AUC', pr_auc)
except (ValueError, AssertionError):
statistics.update_statistics(label, 'PR-AUC', 0.0)
if verbose:
statistics.print_statistics(label)
return statistics
def evaluate_crf_model(x, y, estimator, labels, uniprot=None, verbose=0):
y_pred = np.asarray(estimator.predict(x))
statistics = Statistics()
statistics.update_statistics('all_labels', 'accuracy', estimator.score(x, y))
bin_labels = [0, 1]
for i, l in enumerate(labels):
y_true_binary_l = y[:, i].astype(int)
y_pred_binary_l = y_pred[:, i].astype(int)
label_stats = compute_label_statistics(y_true_binary_l, y_pred_binary_l, labels=bin_labels)
statistics.update_statistics(l, 'Accuracy', accuracy_score(y_true_binary_l, y_pred_binary_l))
statistics.update_statistics(l, 'Specifcity', label_stats[1]['specificity'])
statistics.update_statistics(l, 'Recall', label_stats[1]['sensitivity'])
statistics.update_statistics(l, 'Precision', label_stats[1]['precision'])
statistics.update_statistics(l, 'FDR', label_stats[1]['fdr'])
statistics.update_statistics(l, 'F-Score (beta=0.5)', fbeta_score(
y_true_binary_l, y_pred_binary_l, beta=0.5, labels=bin_labels, average='binary'
))
statistics.update_statistics(l, 'F-Score (beta=1)', fbeta_score(
y_true_binary_l, y_pred_binary_l, beta=1.0, labels=bin_labels, average='binary'
))
try:
roc_auc = roc_auc_score(y_true_binary_l, y_pred_binary_l, average="binary")
statistics.update_statistics(l, 'ROC-AUC', roc_auc)
except (ValueError, AssertionError):
statistics.update_statistics(l, 'ROC-AUC', np.NaN)
try:
pr_auc = average_precision_score(y_true_binary_l, y_pred_binary_l, average="binary")
statistics.update_statistics(l, 'PR-AUC', pr_auc)
except (ValueError, AssertionError):
statistics.update_statistics(l, 'PR-AUC', np.NaN)
if verbose:
for l in labels:
statistics.print_statistics(l)
if uniprot and verbose:
for u, p1, p2 in zip(uniprot, y, y_pred):
print("\t\t\tResult for {} \n\t\t\t\tTrue: \t{} ||| Pred: \t{}".format(u, p1, p2))
return statistics
