def pr_curve(i): label = labels[i] statistics_l = Statistics() print('Doing label {}'.format(label)) for train_idx, valid_idx in folds: rng = np.random.RandomState() rng.seed(seeds[i]) training_fold = developement_df.loc[train_idx, ] training_fold = training_fold.reset_index(drop=True) validation_fold = developement_df.loc[valid_idx, ] validation_fold = validation_fold.reset_index(drop=True) base_estimators = make_classifiers(method, balanced, labels, random_state=rng) # Find the best params, then do a final proper calibration. base_estimator = base_estimators[label] estimator = RandomizedSearchCV( estimator=base_estimator, param_distributions=params, n_iter=60, scoring='f1', cv=3, random_state=rng, error_score=0.0, n_jobs=1, pre_dispatch='2*n_jobs', refit=True ) # Set up the vectorizer for the bag-of-words representation if vectorizer_method == 'tf-idf': vectorizer = TfidfVectorizer( stop_words=['go', '', ' '], binary=binary, lowercase=True, sublinear_tf=False, max_df=1.0, min_df=0 ) vectorizer.fit(training_fold['terms'].values) elif vectorizer_method == 'count': vectorizer = CountVectorizer( stop_words=['go', '', ' '], binary=binary, lowercase=True ) vectorizer.fit(training_fold['terms'].values) # Fit an evaluate the performance of the classifier. x_train = vectorizer.transform(training_fold['terms'].values) y_train = np.asarray(training_fold[label].values, dtype=int) x_valid = vectorizer.transform(validation_fold['terms'].values) y_valid = np.asarray(validation_fold[label].values, dtype=int) estimator.fit(x_train, y_train) for t in thresholds: y_pred = [int(p[1] >= t) for p in estimator.predict_proba(x_valid)] precision = precision_score(y_valid, y_pred, labels=[0, 1], pos_label=1) recall = recall_score(y_valid, y_pred, labels=[0, 1], pos_label=1) f1 = f1_score(y_valid, y_pred, labels=[0, 1], pos_label=1) statistics_l.update_statistics(label=t, s_type='Precision', data=precision) statistics_l.update_statistics(label=t, s_type='Recall', data=recall) statistics_l.update_statistics(label=t, s_type='F1-Score', data=f1) statistics_l.frame()['reaction'] = label return statistics_l
def evaluate_crf_model(x, y, estimator, labels, uniprot=None, verbose=0): y_pred = np.asarray(estimator.predict(x)) statistics = Statistics() statistics.update_statistics('all_labels', 'accuracy', estimator.score(x, y)) bin_labels = [0, 1] for i, l in enumerate(labels): y_true_binary_l = y[:, i].astype(int) y_pred_binary_l = y_pred[:, i].astype(int) label_stats = compute_label_statistics(y_true_binary_l, y_pred_binary_l, labels=bin_labels) statistics.update_statistics(l, 'Accuracy', accuracy_score(y_true_binary_l, y_pred_binary_l)) statistics.update_statistics(l, 'Specifcity', label_stats[1]['specificity']) statistics.update_statistics(l, 'Recall', label_stats[1]['sensitivity']) statistics.update_statistics(l, 'Precision', label_stats[1]['precision']) statistics.update_statistics(l, 'FDR', label_stats[1]['fdr']) statistics.update_statistics(l, 'F-Score (beta=0.5)', fbeta_score( y_true_binary_l, y_pred_binary_l, beta=0.5, labels=bin_labels, average='binary' )) statistics.update_statistics(l, 'F-Score (beta=1)', fbeta_score( y_true_binary_l, y_pred_binary_l, beta=1.0, labels=bin_labels, average='binary' )) try: roc_auc = roc_auc_score(y_true_binary_l, y_pred_binary_l, average="binary") statistics.update_statistics(l, 'ROC-AUC', roc_auc) except (ValueError, AssertionError): statistics.update_statistics(l, 'ROC-AUC', np.NaN) try: pr_auc = average_precision_score(y_true_binary_l, y_pred_binary_l, average="binary") statistics.update_statistics(l, 'PR-AUC', pr_auc) except (ValueError, AssertionError): statistics.update_statistics(l, 'PR-AUC', np.NaN) if verbose: for l in labels: statistics.print_statistics(l) if uniprot and verbose: for u, p1, p2 in zip(uniprot, y, y_pred): print("\t\t\tResult for {} \n\t\t\t\tTrue: \t{} ||| Pred: \t{}".format(u, p1, p2)) return statistics
def multi_label_evaluate(y, y_prob, threshold): statistics = Statistics() y_pred = (y_prob >= threshold).astype(int) y_pred_50 = (y_prob >= 0.5).astype(int) ranking_loss = label_ranking_loss(y, y_pred) lraps = label_ranking_average_precision_score(y, y_pred) ranking_loss_50 = label_ranking_loss(y, y_pred_50) lraps_50 = label_ranking_average_precision_score(y, y_pred_50) f1_macro = f1_score(y, y_pred, average='macro') f1_macro_50 = f1_score(y, y_pred_50, average='macro') statistics.update_statistics("Multi-Label", "Ranking Loss", ranking_loss) statistics.update_statistics("Multi-Label", "Ranking Precision", lraps) statistics.update_statistics("Multi-Label", "Ranking Loss (t=0.5)", ranking_loss_50) statistics.update_statistics("Multi-Label", "Ranking Precision (t=0.5)", lraps_50) statistics.update_statistics("Multi-Label", "Macro F1", f1_macro) statistics.update_statistics("Multi-Label", "Macro F1 (t=0.5)", f1_macro_50) try: auc_macro = roc_auc_score(y, y_pred, average='macro') auc_macro_50 = roc_auc_score(y, y_pred_50, average='macro') auc_pr_macro = roc_auc_score(y, y_prob, average='macro') statistics.update_statistics("Multi-Label", "Macro AUC", auc_macro) statistics.update_statistics("Multi-Label", "Macro AUC (t=0.5)", auc_macro_50) statistics.update_statistics("Multi-Label", "Macro AUC (Pr)", auc_pr_macro) except ValueError: statistics.update_statistics("Multi-Label", "Macro AUC", np.NaN) statistics.update_statistics("Multi-Label", "Macro AUC (t=0.5)", np.NaN) statistics.update_statistics("Multi-Label", "Macro AUC (Pr)", np.NaN) return statistics
def evaluate_model(y, y_prob, label, threshold): statistics = Statistics() y_pred = (y_prob[:, 1] >= threshold).astype(int) bin_labels = [0, 1] label_stats = compute_label_statistics(y, y_pred, labels=bin_labels) statistics.update_statistics(label, 'F (beta=1)', f1_score(y, y_pred, average='binary', labels=[0,1], pos_label=1)) statistics.update_statistics(label, 'Specificity', label_stats[1]['specificity']) statistics.update_statistics(label, 'Recall', label_stats[1]['sensitivity']) statistics.update_statistics(label, 'Precision', label_stats[1]['precision']) statistics.update_statistics(label, 'FDR', label_stats[1]['fdr']) try: statistics.update_statistics(label, 'ROC-AUC', roc_auc_score(y, y_pred, average='weighted')) except (ValueError, AssertionError): statistics.update_statistics(label, 'ROC-AUC', 0.0) try: pr_auc = average_precision_score(y, y_pred, average='weighted') if str(pr_auc) == 'nan': pr_auc = 0.0 statistics.update_statistics(label, 'PR-AUC', pr_auc) except (ValueError, AssertionError): statistics.update_statistics(label, 'PR-AUC', 0.0) return statistics