def predict_proba(self, X): """The returned estimates for all classes are ordered by the label of classes. Args: X (pandas.DataFrame): Probability estimates of the targets as returned by a ``predict_proba()`` call or equivalent. Note: must include protected attributes in the index. Returns: numpy.ndarray: Returns the probability of the sample for each class in the model, where classes are ordered as they are in ``self.classes_``. """ check_is_fitted(self, 'mix_rates_') rng = check_random_state(self.random_state) groups, _ = check_groups(X, self.prot_attr_) if not set(np.unique(groups)) <= set(self.groups_): raise ValueError('The protected groups from X:\n{}\ndo not ' 'match those from the training set:\n{}'.format( np.unique(groups), self.groups_)) pos_idx = np.nonzero(self.classes_ == self.pos_label_)[0][0] X = X.iloc[:, pos_idx] yt = np.empty_like(X) for grp_idx in range(2): i = (groups == self.groups_[grp_idx]) to_replace = (rng.rand(sum(i)) < self.mix_rates_[grp_idx]) new_preds = X[i].copy() new_preds[to_replace] = self.base_rates_[grp_idx] yt[i] = new_preds return np.c_[1 - yt, yt] if pos_idx == 1 else np.c_[yt, 1 - yt]
def average_odds_error(y_true, y_pred, prot_attr=None, pos_label=1, sample_weight=None): r"""A relaxed version of equality of odds. Returns the average of the absolute difference in FPR and TPR for the unprivileged and privileged groups: .. math:: \dfrac{|FPR_{D = \text{unprivileged}} - FPR_{D = \text{privileged}}| + |TPR_{D = \text{unprivileged}} - TPR_{D = \text{privileged}}|}{2} A value of 0 indicates equality of odds. Args: y_true (pandas.Series): Ground truth (correct) target values. y_pred (array-like): Estimated targets as returned by a classifier. prot_attr (array-like, keyword-only): Protected attribute(s). If ``None``, all protected attributes in y_true are used. priv_group (scalar, optional): The label of the privileged group. pos_label (scalar, optional): The label of the positive class. sample_weight (array-like, optional): Sample weights. Returns: float: Average odds error. """ priv_group = check_groups(y_true, prot_attr=prot_attr)[0][0] fpr_diff = -difference(specificity_score, y_true, y_pred, prot_attr=prot_attr, priv_group=priv_group, pos_label=pos_label, sample_weight=sample_weight) tpr_diff = difference(recall_score, y_true, y_pred, prot_attr=prot_attr, priv_group=priv_group, pos_label=pos_label, sample_weight=sample_weight) return (abs(tpr_diff) + abs(fpr_diff)) / 2
def fit_transform(self, X_train, y_train, X_test): """Remove bias from the given dataset by fair adaptation. Args: X_train (pandas.DataFrame): Training data frame (including the protected attribute). y_train (pandas.Series): Training labels. X_test (pandas.DataFrame): Test data frame (including the protected attribute). Returns: tuple: Transformed inputs. * **X_fair_train** (pandas.DataFrame) -- Transformed training data. * **y_fair_train** (array-like) -- Transformed training labels. * **X_fair_test** (pandas.DataFrame) -- Transformed test data. """ # merge X_train and y_train df_train = pd.concat([X_train, y_train], axis=1) groups, self.prot_attr_ = check_groups(X_train, self.prot_attr, ensure_binary=True) self.groups_ = np.unique(groups) wrapper = osp.join(osp.dirname(osp.abspath(__file__)), 'fairadapt.R') robjects.r.source(wrapper) FairAdapt_R = robjects.r['wrapper'] # convert to Pandas with a local converter with localconverter(robjects.default_converter + pandas2ri.converter): train_data = robjects.conversion.py2rpy(df_train) test_data = robjects.conversion.py2rpy(X_test) adj_mat = robjects.conversion.py2rpy(self.adj_mat) # run FairAdapt in R res = FairAdapt_R(train_data=train_data, test_data=test_data, adj_mat=adj_mat, prot_attr=self.prot_attr_, outcome=y_train.name) with localconverter(robjects.default_converter + pandas2ri.converter): X_fair_train = robjects.conversion.rpy2py(res.rx2('train')) X_fair_test = robjects.conversion.rpy2py(res.rx2('test')) X_fair_train.columns = [y_train.name] + X_train.columns.tolist() y_fair_train = X_fair_train.pop(y_train.name) X_fair_test.columns = X_test.columns return X_fair_train, y_fair_train, X_fair_test
def ratio(func, y, *args, prot_attr=None, priv_group=1, sample_weight=None, **kwargs): """Compute the ratio between unprivileged and privileged subsets for an arbitrary metric. Note: The optimal value of a ratio is 1. To make it a scorer, one must take the minimum of the ratio and its inverse. Unprivileged group is taken to be the inverse of the privileged group. Args: func (function): A metric function from :mod:`sklearn.metrics` or :mod:`aif360.sklearn.metrics.metrics`. y (pandas.Series): Outcome vector with protected attributes as index. *args: Additional positional args to be passed through to func. prot_attr (array-like, keyword-only): Protected attribute(s). If ``None``, all protected attributes in y are used. priv_group (scalar, optional): The label of the privileged group. sample_weight (array-like, optional): Sample weights passed through to func. **kwargs: Additional keyword args to be passed through to func. Returns: scalar: Ratio of metric values for unprivileged and privileged groups. """ groups, _ = check_groups(y, prot_attr) idx = (groups == priv_group) unpriv = map(lambda a: a[~idx], (y, ) + args) priv = map(lambda a: a[idx], (y, ) + args) if sample_weight is not None: numerator = func(*unpriv, sample_weight=sample_weight[~idx], **kwargs) denominator = func(*priv, sample_weight=sample_weight[idx], **kwargs) else: numerator = func(*unpriv, **kwargs) denominator = func(*priv, **kwargs) if denominator == 0: warnings.warn( "The ratio is ill-defined and being set to 0.0 because " "'{}' for privileged samples is 0.".format(func.__name__), UndefinedMetricWarning) return 0. return numerator / denominator
def fit(self, X, y, labels=None, pos_label=1, sample_weight=None): """Compute the mixing rates required to satisfy the cost constraint. Args: X (array-like): Probability estimates of the targets as returned by a ``predict_proba()`` call or equivalent. y (pandas.Series): Ground-truth (correct) target values. labels (list, optional): The ordered set of labels values. Must match the order of columns in X if provided. By default, all labels in y are used in sorted order. pos_label (scalar, optional): The label of the positive class. sample_weight (array-like, optional): Sample weights. Returns: self """ X, y, sample_weight = check_inputs(X, y, sample_weight) groups, self.prot_attr_ = check_groups(y, self.prot_attr, ensure_binary=True) self.classes_ = labels if labels is not None else np.unique(y) self.groups_ = np.unique(groups) self.pos_label_ = pos_label if len(self.classes_) > 2: raise ValueError('Only binary classification is supported.') if pos_label not in self.classes_: raise ValueError('pos_label={} is not in the set of labels. The ' 'valid values are:\n{}'.format(pos_label, self.classes_)) X = X[:, np.nonzero(self.classes_ == self.pos_label_)[0][0]] # local function to return corresponding args for metric evaluation def _args(grp_idx, triv=False): idx = (groups == self.groups_[grp_idx]) pred = np.full_like(X, self.base_rates_[grp_idx]) if triv else X return [y[idx], pred[idx], pos_label, sample_weight[idx]] self.base_rates_ = [base_rate(*_args(i)) for i in range(2)] costs = [self._weighted_cost(*_args(i)) for i in range(2)] self.mix_rates_ = [(costs[1] - costs[0]) / (self._weighted_cost(*_args(0, True)) - costs[0]), (costs[0] - costs[1]) / (self._weighted_cost(*_args(1, True)) - costs[1])] self.mix_rates_[np.argmax(costs)] = 0 return self
def between_group_generalized_entropy_error(y_true, y_pred, prot_attr=None, priv_group=None, alpha=2, pos_label=1): r"""Compute the between-group generalized entropy. Between-group generalized entropy index is proposed as a group fairness measure in [#speicher18]_ and is one of two terms that the generalized entropy index decomposes to. Args: y_true (pandas.Series): Ground truth (correct) target values. y_pred (array-like): Estimated targets as returned by a classifier. prot_attr (array-like, optional): Protected attribute(s). If ``None``, all protected attributes in y_true are used. priv_group (scalar, optional): The label of the privileged group. If provided, the index will be computed between only the privileged and unprivileged groups. Otherwise, the index will be computed between all groups defined by the prot_attr. alpha (scalar, optional): Parameter that regulates the weight given to distances between values at different parts of the distribution. A value of 0 is equivalent to the mean log deviation, 1 is the Theil index, and 2 is half the squared coefficient of variation. pos_label (scalar, optional): The label of the positive class. See also: :func:`generalized_entropy_index` References: .. [#speicher18] `T. Speicher, H. Heidari, N. Grgic-Hlaca, K. P. Gummadi, A. Singla, A. Weller, and M. B. Zafar, "A Unified Approach to Quantifying Algorithmic Unfairness: Measuring Individual and Group Unfairness via Inequality Indices," ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, 2018. <https://dl.acm.org/citation.cfm?id=3220046>`_ """ groups, _ = check_groups(y_true, prot_attr) b = np.empty_like(y_true, dtype='float') if priv_group is not None: groups = [1 if g == priv_group else 0 for g in groups] for g in np.unique(groups): b[groups == g] = (1 + (y_pred[groups == g] == pos_label) - (y_true[groups == g] == pos_label)).mean() return generalized_entropy_index(b, alpha=alpha)
def difference(func, y, *args, prot_attr=None, priv_group=1, sample_weight=None, **kwargs): """Compute the difference between unprivileged and privileged subsets for an arbitrary metric. Note: The optimal value of a difference is 0. To make it a scorer, one must take the absolute value and set greater_is_better to False. Unprivileged group is taken to be the inverse of the privileged group. Args: func (function): A metric function from :mod:`sklearn.metrics` or :mod:`aif360.sklearn.metrics.metrics`. y (pandas.Series): Outcome vector with protected attributes as index. *args: Additional positional args to be passed through to func. prot_attr (array-like, keyword-only): Protected attribute(s). If ``None``, all protected attributes in y are used. priv_group (scalar, optional): The label of the privileged group. sample_weight (array-like, optional): Sample weights passed through to func. **kwargs: Additional keyword args to be passed through to func. Returns: scalar: Difference in metric value for unprivileged and privileged groups. Examples: >>> X, y = fetch_german(numeric_only=True) >>> y_pred = LogisticRegression().fit(X, y).predict(X) >>> difference(precision_score, y, y_pred, prot_attr='sex', ... priv_group='male') -0.06955430006277463 """ groups, _ = check_groups(y, prot_attr) idx = (groups == priv_group) unpriv = map(lambda a: a[~idx], (y, ) + args) priv = map(lambda a: a[idx], (y, ) + args) if sample_weight is not None: return (func(*unpriv, sample_weight=sample_weight[~idx], **kwargs) - func(*priv, sample_weight=sample_weight[idx], **kwargs)) return func(*unpriv, **kwargs) - func(*priv, **kwargs)
def fit_transform(self, X, y, sample_weight=None): """Compute the factors for reweighing the dataset and transform the sample weights. Args: X (pandas.DataFrame): Training samples. y (array-like): Training labels. sample_weight (array-like, optional): Sample weights. Returns: tuple: Samples and their weights. * **X** -- Unchanged samples. * **sample_weight** -- Transformed sample weights. """ X, y, sample_weight = check_inputs(X, y, sample_weight) sample_weight_t = np.empty_like(sample_weight) groups, self.prot_attr_ = check_groups(X, self.prot_attr) # TODO: maintain categorical ordering self.groups_ = np.unique(groups) self.classes_ = np.unique(y) n_groups = len(self.groups_) n_classes = len(self.classes_) self.reweigh_factors_ = np.full((n_groups, n_classes), np.nan) def N_(i): return sample_weight[i].sum() N = sample_weight.sum() for i, g in enumerate(self.groups_): for j, c in enumerate(self.classes_): g_and_c = (groups == g) & (y == c) if np.any(g_and_c): W_gc = N_(groups == g) * N_(y == c) / (N * N_(g_and_c)) sample_weight_t[g_and_c] = W_gc * sample_weight[g_and_c] self.reweigh_factors_[i, j] = W_gc return X, sample_weight_t
def fit(self, X, y): """Train the classifier and adversary (if ``debias == True``) with the given training data. Args: X (pandas.DataFrame): Training samples. y (array-like): Training labels. Returns: self """ X, y, _ = check_inputs(X, y) rng = check_random_state(self.random_state) ii32 = np.iinfo(np.int32) s1, s2, s3, s4 = rng.randint(ii32.min, ii32.max, size=4) tf.reset_default_graph() self.sess_ = tf.Session() groups, self.prot_attr_ = check_groups(X, self.prot_attr) le = LabelEncoder() y = le.fit_transform(y) self.classes_ = le.classes_ # BUG: LabelEncoder converts to ndarray which removes tuple formatting groups = groups.map(str) groups = le.fit_transform(groups) self.groups_ = le.classes_ n_classes = len(self.classes_) n_groups = len(self.groups_) # use sigmoid for binary case if n_classes == 2: n_classes = 1 if n_groups == 2: n_groups = 1 n_samples, n_features = X.shape with tf.variable_scope(self.scope_name): # Setup placeholders self.input_ph = tf.placeholder(tf.float32, shape=[None, n_features]) self.prot_attr_ph = tf.placeholder(tf.float32, shape=[None, 1]) self.true_labels_ph = tf.placeholder(tf.float32, shape=[None, 1]) self.keep_prob = tf.placeholder(tf.float32) # Create classifier with tf.variable_scope('classifier_model'): W1 = tf.get_variable( 'W1', [n_features, self.classifier_num_hidden_units], initializer=tf.initializers.glorot_uniform(seed=s1)) b1 = tf.Variable(tf.zeros( shape=[self.classifier_num_hidden_units]), name='b1') h1 = tf.nn.relu(tf.matmul(self.input_ph, W1) + b1) h1 = tf.nn.dropout(h1, rate=1-self.keep_prob, seed=s2) W2 = tf.get_variable( 'W2', [self.classifier_num_hidden_units, n_classes], initializer=tf.initializers.glorot_uniform(seed=s3)) b2 = tf.Variable(tf.zeros(shape=[n_classes]), name='b2') self.classifier_logits_ = tf.matmul(h1, W2) + b2 # Obtain classifier loss if self.classifier_logits_.shape[1] == 1: clf_loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( labels=self.true_labels_ph, logits=self.classifier_logits_)) else: clf_loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( labels=tf.squeeze(tf.cast(self.true_labels_ph, tf.int32)), logits=self.classifier_logits_)) if self.debias: # Create adversary with tf.variable_scope("adversary_model"): c = tf.get_variable('c', initializer=tf.constant(1.0)) s = tf.sigmoid((1 + tf.abs(c)) * self.classifier_logits_) W2 = tf.get_variable('W2', [3, n_groups], initializer=tf.initializers.glorot_uniform(seed=s4)) b2 = tf.Variable(tf.zeros(shape=[n_groups]), name='b2') self.adversary_logits_ = tf.matmul( tf.concat([s, s * self.true_labels_ph, s * (1. - self.true_labels_ph)], axis=1), W2) + b2 # Obtain adversary loss if self.adversary_logits_.shape[1] == 1: adv_loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( labels=self.prot_attr_ph, logits=self.adversary_logits_)) else: adv_loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( labels=tf.squeeze(tf.cast(self.prot_attr_ph, tf.int32)), logits=self.adversary_logits_)) global_step = tf.Variable(0., trainable=False) init_learning_rate = 0.001 if self.adversary_loss_weight is not None: learning_rate = tf.train.exponential_decay(init_learning_rate, global_step, 1000, 0.96, staircase=True) else: learning_rate = tf.train.inverse_time_decay(init_learning_rate, global_step, 1000, 0.1, staircase=True) # Setup optimizers clf_opt = tf.train.AdamOptimizer(learning_rate) if self.debias: adv_opt = tf.train.AdamOptimizer(learning_rate) clf_vars = [var for var in tf.trainable_variables() if 'classifier_model' in var.name] if self.debias: adv_vars = [var for var in tf.trainable_variables() if 'adversary_model' in var.name] # Compute grad wrt classifier parameters adv_grads = {var: grad for (grad, var) in adv_opt.compute_gradients(adv_loss, var_list=clf_vars)} normalize = lambda x: x / (tf.norm(x) + np.finfo(np.float32).tiny) clf_grads = [] for (grad, var) in clf_opt.compute_gradients(clf_loss, var_list=clf_vars): if self.debias: unit_adv_grad = normalize(adv_grads[var]) # proj_{adv_grad} clf_grad: grad -= tf.reduce_sum(grad * unit_adv_grad) * unit_adv_grad if self.adversary_loss_weight is not None: grad -= self.adversary_loss_weight * adv_grads[var] else: grad -= tf.sqrt(global_step) * adv_grads[var] clf_grads.append((grad, var)) clf_min = clf_opt.apply_gradients(clf_grads, global_step=global_step) if self.debias: with tf.control_dependencies([clf_min]): adv_min = adv_opt.minimize(adv_loss, var_list=adv_vars) self.sess_.run(tf.global_variables_initializer()) # Begin training for epoch in range(self.num_epochs): shuffled_ids = rng.permutation(n_samples) for i in range(n_samples // self.batch_size): batch_ids = shuffled_ids[self.batch_size * i: self.batch_size * (i+1)] batch_features = X.iloc[batch_ids] batch_labels = y[batch_ids][:, np.newaxis] batch_prot_attr = groups[batch_ids][:, np.newaxis] batch_feed_dict = {self.input_ph: batch_features, self.true_labels_ph: batch_labels, self.prot_attr_ph: batch_prot_attr, self.keep_prob: 0.8} if self.debias: _, _, clf_loss_val, adv_loss_val = self.sess_.run( [clf_min, adv_min, clf_loss, adv_loss], feed_dict=batch_feed_dict) if i % 200 == 0 and self.verbose: print("epoch {:>3d}; iter: {:>4d}; batch classifier" " loss: {:.4f}; batch adversarial loss: " "{:.4f}".format(epoch, i, clf_loss_val, adv_loss_val)) else: _, clf_loss_val = self.sess_.run([clf_min, clf_loss], feed_dict=batch_feed_dict) if i % 200 == 0 and self.verbose: print("epoch {:>3d}; iter: {:>4d}; batch classifier" " loss: {:.4f}".format(epoch, i, clf_loss_val)) return self