def get_score(predictions: Union[List, np.ndarray, pd.DataFrame],
is_member: Union[List, np.ndarray, pd.DataFrame],
membership_label: Union[str, float, int] = 1) -> float:
"""
Disparate Impact is the ratio of predictions for a "positive" outcome in a binary classification task
between members of group 1 and group 2, respectively.
.. math::
\\frac{Pr(\\hat{Y} = 1 | D = \\text{group 1})}
{Pr(\\hat{Y} = 1 | D = \\text{group 2})}
Parameters
----------
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Disparate impact between groups.
"""
# Logic to check input types
check_inputs_validity(predictions=predictions, is_member=is_member)
# List needs to be converted to numpy for indexing
is_member = check_and_convert_list_types(is_member)
predictions = check_and_convert_list_types(predictions)
# Identify groups based on membership label
group_2_predictions, group_1_predictions, group_2_group, group_1_group = \
split_array_based_on_membership_label(predictions, is_member, membership_label)
if (group_1_predictions == 1).sum() == 0 and (group_2_predictions == 1).sum() == 0:
warnings.warn("No positive predictions in the dataset, cannot calculate Disparate Impact.")
return np.nan
# Handle division by zero when no positive cases in the group 2 group
if (group_2_predictions == 1).sum() == 0:
warnings.warn(
"No positive predictions found in the group 2 group. Double-check your model works correctly.")
return (group_1_predictions == 1).sum()
# Calculate percentages of positive predictions stratified by group membership
group_1_predictions_pos_ratio = np.sum(group_1_predictions == 1) / len(group_1_group)
group_2_predictions_pos_ratio = np.sum(group_2_predictions == 1) / len(group_2_group)
return group_1_predictions_pos_ratio / group_2_predictions_pos_ratio
def get_score(self, labels: Union[List, np.ndarray, pd.Series],
predictions: Union[List, np.ndarray, pd.Series]) -> float:
"""
The Theil index is the generalized entropy index with :math:`\\alpha = 1`.
See Generalized Entropy index.
Parameters
----------
labels: Union[List, np.ndarray, pd.Series]
Binary ground truth labels for the provided dataset (0/1).
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
Returns
----------
Theil Index of the classifier.
"""
check_input_type(labels)
check_input_type(predictions)
# Check input shape
check_input_shape(labels)
check_input_shape(predictions)
# Check input content
check_binary(labels)
check_binary(predictions)
# Check the actual contents of the arrays
check_elementwise_input_type(labels)
check_elementwise_input_type(predictions)
# Check that our arrays are all the same length
check_true(
len(labels) == len(predictions),
AssertionError(
"Shapes of inputs do not match. You supplied labels :"
f"{len(labels)} and predictions: {len(predictions)}"))
# Convert lists
y_pred = check_and_convert_list_types(predictions)
y_true = check_and_convert_list_types(labels)
y_pred = (y_pred == self.positive_label_name).astype(np.float64)
y_true = (y_true == self.positive_label_name).astype(np.float64)
b = 1 + y_pred - y_true
return float(np.mean(np.log((b / np.mean(b))**b) / np.mean(b)))
def get_score(predictions: Union[List, np.ndarray, pd.Series],
is_member: Union[List, np.ndarray, pd.Series],
membership_label: Union[str, float, int] = 1) -> float:
"""
Difference in statistical parity between two groups.
.. math::
P(Y_{hat}=1 | group = \\text{group 1} ) - P(Y_{hat} = 1 | \\text{group 2})
Parameters
----------
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Statistical parity difference between groups.
"""
# Check input types
check_inputs_validity(predictions=predictions, is_member=is_member)
# Convert lists to numpy arrays
is_member = check_and_convert_list_types(is_member)
predictions = check_and_convert_list_types(predictions)
# Identify the group 2 and group 1 group based on specified group label
group_2_predictions, group_1_predictions, group_2_group, group_1_group = \
split_array_based_on_membership_label(predictions, is_member, membership_label)
group_1_predictions_pct = np.sum(group_1_predictions == 1) / len(group_1_group)
group_2_predictions_pct = np.sum(group_2_predictions == 1) / len(group_2_group)
return group_1_predictions_pct - group_2_predictions_pct
def get_score(labels: Union[List, np.ndarray, pd.Series],
predictions: Union[List, np.ndarray, pd.Series],
is_member: Union[List, np.ndarray, pd.Series],
membership_label: Union[str, float, int] = 1) -> float:
"""
We define the predictive equality as the situation when accuracy of decisions is equal across race groups,
as measured by false positive rate (FPR).
Drawing the analogy of gender classification where race is the protected attribute, across all race groups,
the ratio of men incorrectly predicted to be a woman is the same.
More formally,
.. math::
E[d(X)|Y=0, g(X)] = E[d(X), Y=0]
Parameters
----------
labels: Union[List, np.ndarray, pd.Series]
Binary ground truth labels for the provided dataset (0/1).
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Predictive Equality difference between groups.
"""
# Check input types
check_inputs_validity(labels=labels,
predictions=predictions,
is_member=is_member,
optional_labels=False)
# Convert to numpy arrays
is_member = check_and_convert_list_types(is_member)
predictions = check_and_convert_list_types(predictions)
labels = check_and_convert_list_types(labels)
# Identify the group 1 and group 2 based on membership label
group_2_truth, group_1_truth, group_2_group_idx, group_1_group_idx = \
split_array_based_on_membership_label(labels, is_member, membership_label)
if np.unique(group_2_truth).shape[0] == 1 or np.unique(
group_1_truth).shape[0] == 1:
return warnings.warn(
"Encountered homogeneous unary ground truth either in group 2/group 1 group. \
Predictive Equality cannot be calculated.")
fpr_group_1 = performance_measures(labels,
predictions,
group_1_group_idx,
group_membership=True)["FPR"]
fpr_group_2 = performance_measures(labels,
predictions,
group_2_group_idx,
group_membership=True)["FPR"]
return fpr_group_1 - fpr_group_2
def get_score(labels: Union[List, np.ndarray, pd.Series],
predictions: Union[List, np.ndarray, pd.Series],
is_member: Union[List, np.ndarray, pd.Series],
membership_label: Union[str, float, int] = 1) -> float:
"""Calculate the ratio of true positives to positive examples in the dataset, :math:`TPR = TP/P`,
conditioned on a protected attribute.
Parameters
----------
labels: Union[List, np.ndarray, pd.Series]
Binary ground truth labels for the provided dataset (0/1).
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Equal opportunity difference between groups.
"""
# Logic to check input types.
check_inputs_validity(predictions=predictions,
is_member=is_member,
optional_labels=False,
labels=labels)
# List needs to be converted to np for indexing
is_member = check_and_convert_list_types(is_member)
predictions = check_and_convert_list_types(predictions)
labels = check_and_convert_list_types(labels)
# Identify the group 2 and group 1 group based on membership label
group_2_group_idx, group_1_group_idx, = \
split_array_based_on_membership_label(None, is_member, membership_label, return_index_only=True)
if np.unique(labels[group_1_group_idx]).shape[0] == 1 or np.unique(
labels[group_2_group_idx]).shape[0] == 1:
warnings.warn(
"Encountered homogeneous unary ground truth either in group 2/group 1 group. \
Equal Opportunity will be calculated but numpy will raise division by zero."
)
elif np.unique(labels[group_1_group_idx]).shape[0] == 1 and \
np.unique(labels[group_2_group_idx]).shape[0] == 1:
warnings.warn(
"Encountered homogeneous unary ground truth in both group 1/group 2. \
Equal Opportunity cannot be calculated.")
tpr_group_1 = performance_measures(labels,
predictions,
group_1_group_idx,
group_membership=True)["TPR"]
tpr_group_2 = performance_measures(labels,
predictions,
group_2_group_idx,
group_membership=True)["TPR"]
return tpr_group_1 - tpr_group_2
def get_score(self,
labels: Union[List, np.ndarray, pd.Series],
predictions: Union[List, np.ndarray, pd.Series],
alpha: float = 2) -> float:
"""Generalized entropy index is proposed as a unified individual and group fairness measure in [3]_.
With :math:`b_i = \\hat{y}_i - y_i + 1`:
.. math::
\\mathcal{E}(\\alpha) = \\begin{cases}
\\frac{1}{n \\alpha (\\alpha-1)}\\sum_{i=1}^n\\left[\\left(\\frac{b_i}{\\mu}\\right)^\\alpha - 1\\right] &
\\alpha \\ne 0, 1, \\\\
\\frac{1}{n}\\sum_{i=1}^n\\frac{b_{i}}{\\mu}\\ln\\frac{b_{i}}{\\mu} & \\alpha=1, \\\\
-\\frac{1}{n}\\sum_{i=1}^n\\ln\\frac{b_{i}}{\\mu},& \\alpha=0.
\\end{cases}
Parameters
----------
labels: Union[List, np.ndarray, pd.Series]
Binary ground truth labels for the provided dataset (0/1).
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
alpha: float
Parameter that regulates weight given to distances between values at different parts of the distribution.
Default value is 2.
Returns
----------
General Entropy Index of the classifier.
References:
----------
.. [3] T. Speicher, H. Heidari, N. Grgic-Hlaca, K. P. Gummadi, A. Singla, A. Weller, and M. B. Zafar,
A Unified Approach to Quantifying Algorithmic Unfairness: Measuring Individual and Group Unfairness via
Inequality Indices, ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, 2018.
"""
# Check input types
check_input_type(labels)
check_input_type(predictions)
# Check input shapes
check_input_shape(labels)
check_input_shape(predictions)
# Check input content
check_binary(labels)
check_binary(predictions)
# Check the actual contents of the arrays
check_elementwise_input_type(labels)
check_elementwise_input_type(predictions)
# Check that our arrays are all the same length
check_true(
len(labels) == len(predictions),
AssertionError(
"Shapes of inputs do not match. You supplied labels :"
f"{len(labels)} and predictions: {len(predictions)}"))
# Convert
y_pred = check_and_convert_list_types(predictions)
y_true = check_and_convert_list_types(labels)
y_pred = (y_pred == self.positive_label_name).astype(np.float64)
y_true = (y_true == self.positive_label_name).astype(np.float64)
b = 1 + y_pred - y_true
if alpha == 1:
# moving the b inside the log allows for 0 values
return float(np.mean(np.log((b / np.mean(b))**b) / np.mean(b)))
elif alpha == 0:
return -np.mean(np.log(b / np.mean(b)))
else:
return np.mean((b / np.mean(b))**alpha - 1) / (alpha * (alpha - 1))
def test_check_and_convert_list_conversion_valid(self):
my_list = [1, 2, 3]
assert type(check_and_convert_list_types(my_list)) == np.ndarray
def get_score(labels: Union[List, np.ndarray, pd.Series],
predictions: Union[List, np.ndarray, pd.Series],
is_member: Union[List, np.ndarray, pd.Series],
membership_label: Union[str, float, int] = 1) -> float:
"""
The average odds denote the average of difference in FPR and TPR for group 1 and group 2.
.. math::
\\frac{1}{2} [(FPR_{D = \\text{group 1}} - FPR_{D =
\\text{group 2}}) + (TPR_{D = \\text{group 2}} - TPR_{D
= \\text{group 1}}))]
If predictions within ANY group are homogeneous, we cannot calculate some of the performance measures
(such as TPR,TNR,FPR,FNR), in this case, NaN is returned.
Parameters
----------
labels: Union[List, np.ndarray, pd.Series]
Binary ground truth labels for the provided dataset (0/1).
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Average odds difference between groups.
"""
# Logic to check input types
check_inputs_validity(predictions=predictions,
is_member=is_member,
optional_labels=False,
labels=labels)
# List needs to be converted to numpy for indexing
is_member = check_and_convert_list_types(is_member)
predictions = check_and_convert_list_types(predictions)
labels = check_and_convert_list_types(labels)
# Identify the group 2 and group 1 group based on membership label
group_2_truth, group_1_truth, group_2_group_idx, group_1_group_idx = \
split_array_based_on_membership_label(labels, is_member, membership_label)
if np.unique(group_2_truth).shape[0] == 1 or np.unique(
group_1_truth).shape[0] == 1:
return warnings.warn(
"Encountered homogeneous unary ground truth either in group 2/group 1 group. "
"Average Odds cannot be calculated.")
results_group_1 = performance_measures(labels,
predictions,
group_idx=group_1_group_idx,
group_membership=True)
results_group_2 = performance_measures(labels,
predictions,
group_idx=group_2_group_idx,
group_membership=True)
fpr_group_1 = results_group_1["FPR"]
fpr_group_2 = results_group_2["FPR"]
tpr_group_1 = results_group_1["TPR"]
tpr_group_2 = results_group_2["TPR"]
return 0.5 * (fpr_group_1 - fpr_group_2) + 0.5 * (tpr_group_1 -
tpr_group_2)