def __fair_regression_measures( y_true: ArrayLike, y_pred: ArrayLike, pa_name: str, priv_grp: int = 1 ): """ Returns dict of regression-specific fairness measures """ def predmean(_, y_pred, *args): return np.mean(y_pred.values) def meanerr(y_true, y_pred, *args): return np.mean((y_pred - y_true).values) # measures = {} # Ratios measures["Mean Prediction Ratio"] = aif.ratio( predmean, y_true, y_pred, prot_attr=pa_name, priv_group=priv_grp ) measures["MAE Ratio"] = aif.ratio( mean_absolute_error, y_true, y_pred, prot_attr=pa_name, priv_group=priv_grp ) # Differences measures["Mean Prediction Difference"] = aif.difference( predmean, y_true, y_pred, prot_attr=pa_name, priv_group=priv_grp ) measures["MAE Difference"] = aif.difference( mean_absolute_error, y_true, y_pred, prot_attr=pa_name, priv_group=priv_grp ) return measures
def __fair_classification_measures( y_true: ArrayLike, y_pred: ArrayLike, pa_name: str, priv_grp: int = 1 ): """ Returns a dict of classification-specific fairness measures """ def predmean(_, y_pred, *args): return np.mean(y_pred.values) # measures = {} measures["Selection Ratio"] = aif.ratio( predmean, y_true, y_pred, prot_attr=pa_name, priv_group=priv_grp ) measures["PPV Ratio"] = fcmtrc.ppv_ratio(y_true, y_pred, pa_name, priv_grp) measures["TPR Ratio"] = fcmtrc.tpr_ratio(y_true, y_pred, pa_name, priv_grp) measures["FPR Ratio"] = fcmtrc.fpr_ratio(y_true, y_pred, pa_name, priv_grp) # measures["Selection Diff"] = aif.difference( predmean, y_true, y_pred, prot_attr=pa_name, priv_group=priv_grp ) measures["PPV Diff"] = fcmtrc.ppv_diff(y_true, y_pred, pa_name, priv_grp) measures["TPR Diff"] = fcmtrc.tpr_diff(y_true, y_pred, pa_name, priv_grp) measures["FPR Diff"] = fcmtrc.fpr_diff(y_true, y_pred, pa_name, priv_grp) measures["Balanced Accuracy Difference"] = aif.difference( balanced_accuracy_score, y_true, y_pred, prot_attr=pa_name, priv_group=priv_grp ) measures["Balanced Accuracy Ratio"] = aif.ratio( balanced_accuracy_score, y_true, y_pred, prot_attr=pa_name, priv_group=priv_grp ) return measures
def __binary_group_fairness_measures(X, prtc_attr, y_true, y_pred, y_prob=None, priv_grp=1): """[summary] Args: X (pandas DataFrame): Sample features prtc_attr (named array-like): values for the protected attribute (note: protected attribute may also be present in X) y_true (pandas DataFrame): Sample targets y_pred (pandas DataFrame): Sample target predictions y_prob (pandas DataFrame, optional): Sample target probabilities. Defaults to None. Returns: [type]: [description] """ pa_names = prtc_attr.columns.tolist() gf_vals = {} gf_key = 'Group Fairness' gf_vals['Statistical Parity Difference'] = \ aif_mtrc.statistical_parity_difference(y_true, y_pred, prot_attr=pa_names) gf_vals['Disparate Impact Ratio'] = \ aif_mtrc.disparate_impact_ratio(y_true, y_pred, prot_attr=pa_names) if not helper.is_tutorial_running() and not len(pa_names) > 1: gf_vals['Demographic Parity Difference'] = \ fl_mtrc.demographic_parity_difference(y_true, y_pred, sensitive_features=prtc_attr) gf_vals['Demographic Parity Ratio'] = \ fl_mtrc.demographic_parity_ratio(y_true, y_pred, sensitive_features=prtc_attr) gf_vals['Average Odds Difference'] = \ aif_mtrc.average_odds_difference(y_true, y_pred, prot_attr=pa_names) gf_vals['Equal Opportunity Difference'] = \ aif_mtrc.equal_opportunity_difference(y_true, y_pred, prot_attr=pa_names) if not helper.is_tutorial_running() and not len(pa_names) > 1: gf_vals['Equalized Odds Difference'] = \ fl_mtrc.equalized_odds_difference(y_true, y_pred, sensitive_features=prtc_attr) gf_vals['Equalized Odds Ratio'] = \ fl_mtrc.equalized_odds_ratio(y_true, y_pred, sensitive_features=prtc_attr) gf_vals['Positive Predictive Parity Difference'] = \ aif_mtrc.difference(sk_metric.precision_score, y_true, y_pred, prot_attr=pa_names, priv_group=priv_grp) gf_vals['Balanced Accuracy Difference'] = \ aif_mtrc.difference(sk_metric.balanced_accuracy_score, y_true, y_pred, prot_attr=pa_names, priv_group=priv_grp) if y_prob is not None: gf_vals['AUC Difference'] = \ aif_mtrc.difference(sk_metric.roc_auc_score, y_true, y_prob, prot_attr=pa_names, priv_group=priv_grp) return (gf_key, gf_vals)
def update_summary( summary_dict: Dict[str, Number], pa_name: str, y_true: ArrayLike, y_pred: ArrayLike, y_prob: ArrayLike, priv_grp: int, ): """ Adds replaces measure keys with the names found in the literature Args: X (pandas DataFrame): Sample features pa_name (str): y_true (pandas DataFrame): Sample targets y_pred (pandas DataFrame): Sample target predictions y_prob (pandas DataFrame, optional): Sample target probabilities. Defaults to None. priv_grp (int): Specifies which label indicates the privileged group. Defaults to 1. """ name_update = { "Selection Diff": "Statistical Parity Difference", "Selection Ratio": "Disparate Impact Ratio", "PPV Diff": "Positive Predictive Parity Difference", "PPV Ratio": "Positive Predictive Parity Ratio", } drop_keys = ["TPR Ratio", "TPR Diff", "FPR Ratio", "FPR Diff"] for k in name_update.keys(): val = summary_dict.pop(k) summary_dict[name_update[k]] = val for k in drop_keys: summary_dict.pop(k) summary_dict["Equal Odds Difference"] = fcmtrc.eq_odds_diff( y_true, y_pred, pa_name=pa_name ) summary_dict["Equal Odds Ratio"] = fcmtrc.eq_odds_ratio( y_true, y_pred, pa_name=pa_name ) if y_prob is not None: try: summary_dict["AUC Difference"] = aif.difference( pmtrc.roc_auc_score, y_true, y_prob, prot_attr=pa_name, priv_group=priv_grp, ) except: pass return summary_dict
def fnr_diff(y_true: pd.Series, y_pred: pd.Series, pa_name: str, priv_grp: int = 1): """ Returns the between-group difference of False Negative Rates Args: y_true (pd.Series): true target values y_pred (pd.Series): predicted target values prtc_attr (str): name of the protected attribute priv_grp (int, optional): . Defaults to 1. Returns: Number """ return difference(false_negative_rate, y_true, y_pred, prot_attr=pa_name, priv_group=priv_grp)
def ppv_diff(y_true: pd.Series, y_pred: pd.Series, pa_name: str, priv_grp: int = 1): """ Returns the between-group difference of Positive Predictive Values Args: y_true (pd.Series): true target values y_pred (pd.Series): predicted target values prtc_attr (str): name of the protected attribute priv_grp (int, optional): . Defaults to 1. Returns: Number """ return difference(precision, y_true, y_pred, prot_attr=pa_name, priv_group=priv_grp)
def score(self, X, y, sample_weight=None): """Score the predictions according to the cost constraint specified. Args: X (pandas.DataFrame): Probability estimates of the targets as returned by a ``predict_proba()`` call or equivalent. Note: must include protected attributes in the index. y (array-like): Ground-truth (correct) target values. sample_weight (array-like, optional): Sample weights. Returns: float: Absolute value of the difference in cost function for the two groups (e.g. :func:`~aif360.sklearn.metrics.generalized_fpr` if ``self.cost_constraint`` is 'fpr') """ check_is_fitted(self, ['classes_', 'pos_label_']) pos_idx = np.nonzero(self.classes_ == self.pos_label_)[0][0] probas_pred = self.predict_proba(X)[:, pos_idx] return abs(difference(self._weighted_cost, y, probas_pred, prot_attr=self.prot_attr_, priv_group=self.groups_[1], pos_label=self.pos_label_, sample_weight=sample_weight))