def test_check_inputs_validity_invalid(self):
labels = np.array([1, 0, 2])
y_pred = np.array([1, 1, 0])
is_member = np.array([0, 0, 1])
with self.assertRaises(ValueError):
check_inputs_validity(y_pred, is_member, False, labels)
def get_scores(
self, predictions: Union[List[List], np.ndarray, pd.Series, List],
is_member: Union[List, np.ndarray, pd.Series]) -> List[float]:
"""
Method to calculate a fairness score for each class.
Parameters
----------
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
Returns
----------
Fairness metric for each class.
"""
check_inputs_validity(predictions,
is_member,
optional_labels=True,
binary_only=False)
one_hot_predictions = self._one_hot_encode_classes(predictions)
scores = []
for class_ in self.list_of_classes:
score = self._binary_score(one_hot_predictions[class_], is_member)
scores.append(score)
return scores
def get_all_scores(labels: Union[List, np.ndarray, pd.Series],
predictions: Union[List, np.ndarray, pd.Series],
is_member: Union[List, np.ndarray, pd.Series],
membership_label: Union[str, float, int] = 1) -> pd.DataFrame:
"""
Calculates and tabulates all of the fairness metric scores.
Parameters
----------
labels: Union[List, np.ndarray, pd.Series]
Binary ground truth labels for the provided dataset (0/1).
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Pandas data frame with all implemented binary fairness metrics.
"""
# Logic to check input types
check_inputs_validity(labels=labels, predictions=predictions, is_member=is_member, optional_labels=False)
fairness_funcs = inspect.getmembers(BinaryFairnessMetrics, predicate=inspect.isclass)[:-1]
df = pd.DataFrame(columns=["Metric", "Value", "Ideal Value", "Lower Bound", "Upper Bound"])
for fairness_func in fairness_funcs:
name = fairness_func[0]
class_ = getattr(BinaryFairnessMetrics, name) # grab a class which is a property of BinaryFairnessMetrics
instance = class_() # dynamically instantiate such class
if name in ["DisparateImpact", "StatisticalParity"]:
score = instance.get_score(predictions, is_member, membership_label)
elif name in ["GeneralizedEntropyIndex", "TheilIndex"]:
score = instance.get_score(labels, predictions)
else:
score = instance.get_score(labels, predictions, is_member, membership_label)
if score is None:
score = np.nan
score = np.round(score, 3)
df = df.append({"Metric": instance.name,
"Value": score,
"Lower Bound": instance.lower_bound,
"Ideal Value": instance.ideal_value,
"Upper Bound": instance.upper_bound}, ignore_index=True)
df = df.set_index("Metric")
return df
def get_score(predictions: Union[List, np.ndarray, pd.DataFrame],
is_member: Union[List, np.ndarray, pd.DataFrame],
membership_label: Union[str, float, int] = 1) -> float:
"""
Disparate Impact is the ratio of predictions for a "positive" outcome in a binary classification task
between members of group 1 and group 2, respectively.
.. math::
\\frac{Pr(\\hat{Y} = 1 | D = \\text{group 1})}
{Pr(\\hat{Y} = 1 | D = \\text{group 2})}
Parameters
----------
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Disparate impact between groups.
"""
# Logic to check input types
check_inputs_validity(predictions=predictions, is_member=is_member)
# List needs to be converted to numpy for indexing
is_member = check_and_convert_list_types(is_member)
predictions = check_and_convert_list_types(predictions)
# Identify groups based on membership label
group_2_predictions, group_1_predictions, group_2_group, group_1_group = \
split_array_based_on_membership_label(predictions, is_member, membership_label)
if (group_1_predictions == 1).sum() == 0 and (group_2_predictions == 1).sum() == 0:
warnings.warn("No positive predictions in the dataset, cannot calculate Disparate Impact.")
return np.nan
# Handle division by zero when no positive cases in the group 2 group
if (group_2_predictions == 1).sum() == 0:
warnings.warn(
"No positive predictions found in the group 2 group. Double-check your model works correctly.")
return (group_1_predictions == 1).sum()
# Calculate percentages of positive predictions stratified by group membership
group_1_predictions_pos_ratio = np.sum(group_1_predictions == 1) / len(group_1_group)
group_2_predictions_pos_ratio = np.sum(group_2_predictions == 1) / len(group_2_group)
return group_1_predictions_pos_ratio / group_2_predictions_pos_ratio
def get_score(predictions: Union[List, np.ndarray, pd.Series],
is_member: Union[List, np.ndarray, pd.Series],
membership_label: Union[str, float, int] = 1) -> float:
"""
Difference in statistical parity between two groups.
.. math::
P(Y_{hat}=1 | group = \\text{group 1} ) - P(Y_{hat} = 1 | \\text{group 2})
Parameters
----------
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Statistical parity difference between groups.
"""
# Check input types
check_inputs_validity(predictions=predictions, is_member=is_member)
# Convert lists to numpy arrays
is_member = check_and_convert_list_types(is_member)
predictions = check_and_convert_list_types(predictions)
# Identify the group 2 and group 1 group based on specified group label
group_2_predictions, group_1_predictions, group_2_group, group_1_group = \
split_array_based_on_membership_label(predictions, is_member, membership_label)
group_1_predictions_pct = np.sum(group_1_predictions == 1) / len(group_1_group)
group_2_predictions_pct = np.sum(group_2_predictions == 1) / len(group_2_group)
return group_1_predictions_pct - group_2_predictions_pct
def get_score(labels: Union[List, np.ndarray, pd.Series],
predictions: Union[List, np.ndarray, pd.Series],
is_member: Union[List, np.ndarray, pd.Series],
membership_label: Union[str, float, int] = 1) -> float:
"""
We define the predictive equality as the situation when accuracy of decisions is equal across race groups,
as measured by false positive rate (FPR).
Drawing the analogy of gender classification where race is the protected attribute, across all race groups,
the ratio of men incorrectly predicted to be a woman is the same.
More formally,
.. math::
E[d(X)|Y=0, g(X)] = E[d(X), Y=0]
Parameters
----------
labels: Union[List, np.ndarray, pd.Series]
Binary ground truth labels for the provided dataset (0/1).
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Predictive Equality difference between groups.
"""
# Check input types
check_inputs_validity(labels=labels,
predictions=predictions,
is_member=is_member,
optional_labels=False)
# Convert to numpy arrays
is_member = check_and_convert_list_types(is_member)
predictions = check_and_convert_list_types(predictions)
labels = check_and_convert_list_types(labels)
# Identify the group 1 and group 2 based on membership label
group_2_truth, group_1_truth, group_2_group_idx, group_1_group_idx = \
split_array_based_on_membership_label(labels, is_member, membership_label)
if np.unique(group_2_truth).shape[0] == 1 or np.unique(
group_1_truth).shape[0] == 1:
return warnings.warn(
"Encountered homogeneous unary ground truth either in group 2/group 1 group. \
Predictive Equality cannot be calculated.")
fpr_group_1 = performance_measures(labels,
predictions,
group_1_group_idx,
group_membership=True)["FPR"]
fpr_group_2 = performance_measures(labels,
predictions,
group_2_group_idx,
group_membership=True)["FPR"]
return fpr_group_1 - fpr_group_2
def get_score(labels: Union[List, np.ndarray, pd.Series],
predictions: Union[List, np.ndarray, pd.Series],
is_member: Union[List, np.ndarray, pd.Series],
membership_label: Union[str, float, int] = 1) -> float:
"""Calculate the ratio of true positives to positive examples in the dataset, :math:`TPR = TP/P`,
conditioned on a protected attribute.
Parameters
----------
labels: Union[List, np.ndarray, pd.Series]
Binary ground truth labels for the provided dataset (0/1).
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Equal opportunity difference between groups.
"""
# Logic to check input types.
check_inputs_validity(predictions=predictions,
is_member=is_member,
optional_labels=False,
labels=labels)
# List needs to be converted to np for indexing
is_member = check_and_convert_list_types(is_member)
predictions = check_and_convert_list_types(predictions)
labels = check_and_convert_list_types(labels)
# Identify the group 2 and group 1 group based on membership label
group_2_group_idx, group_1_group_idx, = \
split_array_based_on_membership_label(None, is_member, membership_label, return_index_only=True)
if np.unique(labels[group_1_group_idx]).shape[0] == 1 or np.unique(
labels[group_2_group_idx]).shape[0] == 1:
warnings.warn(
"Encountered homogeneous unary ground truth either in group 2/group 1 group. \
Equal Opportunity will be calculated but numpy will raise division by zero."
)
elif np.unique(labels[group_1_group_idx]).shape[0] == 1 and \
np.unique(labels[group_2_group_idx]).shape[0] == 1:
warnings.warn(
"Encountered homogeneous unary ground truth in both group 1/group 2. \
Equal Opportunity cannot be calculated.")
tpr_group_1 = performance_measures(labels,
predictions,
group_1_group_idx,
group_membership=True)["TPR"]
tpr_group_2 = performance_measures(labels,
predictions,
group_2_group_idx,
group_membership=True)["TPR"]
return tpr_group_1 - tpr_group_2
def test_check_inputs_validity_valid(self):
labels = np.array([1, 1, 0])
y_pred = np.array([1, 1, 0])
is_member = np.array([0, 0, 1])
check_inputs_validity(y_pred, is_member, False, labels)
def test_check_inputs_validity_missing_label(self):
predictions = [3, 4, 3]
is_member = pd.DataFrame.from_dict({'a': [1, 2, 2]})['a']
check_inputs_validity(labels=None, predictions=predictions, is_member=is_member)
def test_check_inputs_validity_missing_is_member(self):
labels = [0, 1, 1]
predictions = [3, 4, 3]
with self.assertRaises(ValueError):
check_inputs_validity(labels=labels, predictions=predictions, is_member=None)
def test_check_inputs_validity_missing_predictions(self):
labels = [0, 1, 1]
is_member = pd.DataFrame.from_dict({'a': [1, 2, 2]})['a']
with self.assertRaises(ValueError):
check_inputs_validity(labels=labels, predictions=None, is_member=is_member)
def test_check_inputs_validity_missing_all(self):
with self.assertRaises(ValueError):
check_inputs_validity(None, None, None)
def get_score(labels: Union[List, np.ndarray, pd.Series],
predictions: Union[List, np.ndarray, pd.Series],
is_member: Union[List, np.ndarray, pd.Series],
membership_label: Union[str, float, int] = 1) -> float:
"""
The average odds denote the average of difference in FPR and TPR for group 1 and group 2.
.. math::
\\frac{1}{2} [(FPR_{D = \\text{group 1}} - FPR_{D =
\\text{group 2}}) + (TPR_{D = \\text{group 2}} - TPR_{D
= \\text{group 1}}))]
If predictions within ANY group are homogeneous, we cannot calculate some of the performance measures
(such as TPR,TNR,FPR,FNR), in this case, NaN is returned.
Parameters
----------
labels: Union[List, np.ndarray, pd.Series]
Binary ground truth labels for the provided dataset (0/1).
predictions: Union[List, np.ndarray, pd.Series]
Binary predictions from some black-box classifier (0/1).
is_member: Union[List, np.ndarray, pd.Series]
Binary membership labels (0/1).
membership_label: Union[str, float, int]
Value indicating group membership.
Default value is 1.
Returns
----------
Average odds difference between groups.
"""
# Logic to check input types
check_inputs_validity(predictions=predictions,
is_member=is_member,
optional_labels=False,
labels=labels)
# List needs to be converted to numpy for indexing
is_member = check_and_convert_list_types(is_member)
predictions = check_and_convert_list_types(predictions)
labels = check_and_convert_list_types(labels)
# Identify the group 2 and group 1 group based on membership label
group_2_truth, group_1_truth, group_2_group_idx, group_1_group_idx = \
split_array_based_on_membership_label(labels, is_member, membership_label)
if np.unique(group_2_truth).shape[0] == 1 or np.unique(
group_1_truth).shape[0] == 1:
return warnings.warn(
"Encountered homogeneous unary ground truth either in group 2/group 1 group. "
"Average Odds cannot be calculated.")
results_group_1 = performance_measures(labels,
predictions,
group_idx=group_1_group_idx,
group_membership=True)
results_group_2 = performance_measures(labels,
predictions,
group_idx=group_2_group_idx,
group_membership=True)
fpr_group_1 = results_group_1["FPR"]
fpr_group_2 = results_group_2["FPR"]
tpr_group_1 = results_group_1["TPR"]
tpr_group_2 = results_group_2["TPR"]
return 0.5 * (fpr_group_1 - fpr_group_2) + 0.5 * (tpr_group_1 -
tpr_group_2)
