def evaluate_model(y, y_prob, label, threshold): statistics = Statistics() y_pred = (y_prob[:, 1] >= threshold).astype(int) bin_labels = [0, 1] label_stats = compute_label_statistics(y, y_pred, labels=bin_labels) statistics.update_statistics(label, 'F (beta=1)', f1_score(y, y_pred, average='binary', labels=[0,1], pos_label=1)) statistics.update_statistics(label, 'Specificity', label_stats[1]['specificity']) statistics.update_statistics(label, 'Recall', label_stats[1]['sensitivity']) statistics.update_statistics(label, 'Precision', label_stats[1]['precision']) statistics.update_statistics(label, 'FDR', label_stats[1]['fdr']) try: statistics.update_statistics(label, 'ROC-AUC', roc_auc_score(y, y_pred, average='weighted')) except (ValueError, AssertionError): statistics.update_statistics(label, 'ROC-AUC', 0.0) try: pr_auc = average_precision_score(y, y_pred, average='weighted') if str(pr_auc) == 'nan': pr_auc = 0.0 statistics.update_statistics(label, 'PR-AUC', pr_auc) except (ValueError, AssertionError): statistics.update_statistics(label, 'PR-AUC', 0.0) return statistics
def evaluate_model(y, y_pred, y_pred_prob, label, statistics, uniprot=None, verbose=0): y_pred_prob_1 = [x[1] for x in y_pred_prob] if uniprot: for u, p1, p2 in zip(uniprot, y, y_pred_prob_1): print("\t\t\tResult for {}, {} \n\t\t\t\tTrue: \t{} ||| Pred: \t{}".format(label, u, p1, p2)) label_stats = compute_label_statistics(y, y_pred, labels=labels) statistics.update_statistics(label, 'Accuracy', accuracy_score(y, y_pred)) statistics.update_statistics(label, 'F (beta=0.5)', fbeta_score(y, y_pred, beta=0.5, labels=[0, 1], average='binary')) statistics.update_statistics(label, 'F (beta=1)', fbeta_score(y, y_pred, beta=1.0, labels=[0, 1], average='binary')) statistics.update_statistics(label, 'Specificity', label_stats[1]['specificity']) statistics.update_statistics(label, 'Recall', label_stats[1]['sensitivity']) statistics.update_statistics(label, 'Precision', label_stats[1]['precision']) statistics.update_statistics(label, 'FDR', label_stats[1]['fdr']) try: statistics.update_statistics(label, 'ROC-AUC', roc_auc_score(y, y_pred, average="weighted")) except (ValueError, AssertionError): statistics.update_statistics(label, 'AUC', 0.0) try: pr_auc = average_precision_score(y, y_pred, average="weighted") if str(pr_auc) == 'nan': pr_auc = 0.0 statistics.update_statistics(label, 'PR-AUC', pr_auc) except (ValueError, AssertionError): statistics.update_statistics(label, 'PR-AUC', 0.0) if verbose: statistics.print_statistics(label) return statistics
def evaluate_crf_model(x, y, estimator, labels, uniprot=None, verbose=0): y_pred = np.asarray(estimator.predict(x)) statistics = Statistics() statistics.update_statistics('all_labels', 'accuracy', estimator.score(x, y)) bin_labels = [0, 1] for i, l in enumerate(labels): y_true_binary_l = y[:, i].astype(int) y_pred_binary_l = y_pred[:, i].astype(int) label_stats = compute_label_statistics(y_true_binary_l, y_pred_binary_l, labels=bin_labels) statistics.update_statistics(l, 'Accuracy', accuracy_score(y_true_binary_l, y_pred_binary_l)) statistics.update_statistics(l, 'Specifcity', label_stats[1]['specificity']) statistics.update_statistics(l, 'Recall', label_stats[1]['sensitivity']) statistics.update_statistics(l, 'Precision', label_stats[1]['precision']) statistics.update_statistics(l, 'FDR', label_stats[1]['fdr']) statistics.update_statistics(l, 'F-Score (beta=0.5)', fbeta_score( y_true_binary_l, y_pred_binary_l, beta=0.5, labels=bin_labels, average='binary' )) statistics.update_statistics(l, 'F-Score (beta=1)', fbeta_score( y_true_binary_l, y_pred_binary_l, beta=1.0, labels=bin_labels, average='binary' )) try: roc_auc = roc_auc_score(y_true_binary_l, y_pred_binary_l, average="binary") statistics.update_statistics(l, 'ROC-AUC', roc_auc) except (ValueError, AssertionError): statistics.update_statistics(l, 'ROC-AUC', np.NaN) try: pr_auc = average_precision_score(y_true_binary_l, y_pred_binary_l, average="binary") statistics.update_statistics(l, 'PR-AUC', pr_auc) except (ValueError, AssertionError): statistics.update_statistics(l, 'PR-AUC', np.NaN) if verbose: for l in labels: statistics.print_statistics(l) if uniprot and verbose: for u, p1, p2 in zip(uniprot, y, y_pred): print("\t\t\tResult for {} \n\t\t\t\tTrue: \t{} ||| Pred: \t{}".format(u, p1, p2)) return statistics