def test_specific_metrics(self): y_t = [0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1] y_p = [1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0] s_f = [0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1] exp_acc = accuracy_score_group_summary(y_t, y_p, sensitive_features=s_f) exp_roc = roc_auc_score_group_summary(y_t, y_p, sensitive_features=s_f) predictions = {"some model": y_p} sensitive_feature = {"my sf": s_f} actual = _create_group_metric_set(y_t, predictions, sensitive_feature, 'binary_classification') # Do some sanity checks validate_dashboard_dictionary(actual) assert actual['trueY'] == y_t assert actual['predictedY'][0] == y_p assert actual['precomputedFeatureBins'][0]['binVector'] == s_f assert len(actual['precomputedMetrics'][0][0]) == 11 # Cross check the two metrics we computed # Comparisons simplified because s_f was already {0,1} actual_acc = actual['precomputedMetrics'][0][0]['accuracy_score'] assert actual_acc['global'] == exp_acc.overall assert actual_acc['bins'] == list(exp_acc.by_group.values()) actual_roc = actual['precomputedMetrics'][0][0][ 'balanced_accuracy_score'] assert actual_roc['global'] == exp_roc.overall assert actual_roc['bins'] == list(exp_roc.by_group.values())
def test_group_roc_auc_score_max_fpr(): result = metrics.roc_auc_score_group_summary(Y_true, Y_pred, sensitive_features=groups, max_fpr=0.5) expected_overall = skm.roc_auc_score(Y_true, Y_pred, max_fpr=0.5) assert expected_overall == result.overall
def test_group_roc_auc_score_average(): result = metrics.roc_auc_score_group_summary(Y_true, Y_pred, sensitive_features=groups, average='samples') expected_overall = skm.roc_auc_score(Y_true, Y_pred, average='samples') assert expected_overall == result.overall