def _compute_variances(ground_truth_data,
representation_function,
batch_size,
random_state,
eval_batch_size=64):
"""Computes the variance for each dimension of the representation.
Args:
ground_truth_data: GroundTruthData to be sampled from.
representation_function: Function that takes observation as input and
outputs a representation.
batch_size: Number of points to be used to compute the variances.
random_state: Numpy random state used for randomness.
eval_batch_size: Batch size used to eval representation.
Returns:
Vector with the variance of each dimension.
"""
observations = ground_truth_data.sample_observations(
batch_size, random_state)
representations = utils.obtain_representation(observations,
representation_function,
eval_batch_size)
representations = np.transpose(representations)
assert representations.shape[0] == batch_size
return np.var(representations, axis=0, ddof=1)
def compute_unified_score_on_fixed_data(
observations, labels, representation_function,
train_percentage=gin.REQUIRED, matrix_fns=gin.REQUIRED, batch_size=100):
"""Computes the unified scores on the fixed set of observations and labels.
Args:
observations: Observations on which to compute the score. Observations have
shape (num_observations, 64, 64, num_channels).
labels: Observed factors of variations.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
train_percentage: Percentage of observations used for training.
matrix_fns: List of functions to relate factors of variations and codes.
batch_size: Batch size used to compute the representation.
Returns:
Unified scores.
"""
mus = utils.obtain_representation(observations, representation_function,
batch_size)
assert labels.shape[1] == observations.shape[0], "Wrong labels shape."
assert mus.shape[1] == observations.shape[0], "Wrong representation shape."
mus_train, mus_test = utils.split_train_test(
mus,
train_percentage)
ys_train, ys_test = utils.split_train_test(
labels,
train_percentage)
return unified_scores(mus_train, ys_train, mus_test, ys_test, matrix_fns)
def compute_sap_on_fixed_data(observations, labels, representation_function,
train_percentage=gin.REQUIRED,
continuous_factors=gin.REQUIRED,
batch_size=100):
"""Computes the SAP score on the fixed set of observations and labels.
Args:
observations: Observations on which to compute the score. Observations have
shape (num_observations, 64, 64, num_channels).
labels: Observed factors of variations.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
train_percentage: Percentage of observations used for training.
continuous_factors: Whether factors should be considered continuous or
discrete.
batch_size: Batch size used to compute the representation.
Returns:
SAP computed on the provided observations and labels.
"""
mus = utils.obtain_representation(observations, representation_function,
batch_size)
assert labels.shape[1] == observations.shape[0], "Wrong labels shape."
assert mus.shape[1] == observations.shape[0], "Wrong representation shape."
mus_train, mus_test = utils.split_train_test(
mus,
train_percentage)
ys_train, ys_test = utils.split_train_test(
labels,
train_percentage)
return _compute_sap(mus_train, ys_train, mus_test, ys_test,
continuous_factors)
def compute_mig_on_fixed_data(observations, labels, representation_function,
batch_size=100):
"""Computes the MIG scores on the fixed set of observations and labels.
Args:
observations: Observations on which to compute the score. Observations have
shape (num_observations, 64, 64, num_channels).
labels: Observed factors of variations.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
batch_size: Batch size used to compute the representation.
Returns:
MIG computed on the provided observations and labels.
"""
mus = utils.obtain_representation(observations, representation_function,
batch_size)
assert labels.shape[1] == observations.shape[0], "Wrong labels shape."
assert mus.shape[1] == observations.shape[0], "Wrong representation shape."
return _compute_mig(mus, labels)
def compute_fairness(ground_truth_data,
representation_function,
random_state,
artifact_dir=None,
num_train=gin.REQUIRED,
num_test_points_per_class=gin.REQUIRED,
batch_size=16):
"""Computes unfairness scores.
We first compute either the mean or maximum total variation for a given
sensitive and target variable. Then, we either average or take the maximum
with respect to target and sensitive variable. For convenience, we compute and
save all combinations. The score used in Section 4 of the paper is here called
mean_fairness:mean_pred:mean_sens.
Args:
ground_truth_data: GroundTruthData to be sampled from.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
random_state: Numpy random state used for randomness.
artifact_dir: Optional path to directory where artifacts can be saved.
num_train: Number of points used for training.
num_test_points_per_class: Number of points used for testing.
batch_size: Batch size for sampling.
Returns:
Dictionary with scores.
"""
del artifact_dir
factor_counts = ground_truth_data.factors_num_values
num_factors = len(factor_counts)
scores = {}
# Training a predictive model.
mus_train, ys_train = utils.generate_batch_factor_code(
ground_truth_data, representation_function, num_train, random_state,
batch_size)
predictor_model_fn = utils.make_predictor_fn()
# For each factor train a single predictive model.
mean_fairness = np.zeros((num_factors, num_factors), dtype=np.float64)
max_fairness = np.zeros((num_factors, num_factors), dtype=np.float64)
for i in range(num_factors):
model = predictor_model_fn()
model.fit(np.transpose(mus_train), ys_train[i, :])
for j in range(num_factors):
if i == j:
continue
# Sample a random set of factors once.
original_factors = ground_truth_data.sample_factors(
num_test_points_per_class, random_state)
counts = np.zeros((factor_counts[i], factor_counts[j]),
dtype=np.int64)
for c in range(factor_counts[j]):
# Intervene on the sensitive attribute.
intervened_factors = np.copy(original_factors)
intervened_factors[:, j] = c
# Obtain the batched observations.
observations = ground_truth_data.sample_observations_from_factors(
intervened_factors, random_state)
representations = utils.obtain_representation(
observations, representation_function, batch_size)
# Get the predictions.
predictions = model.predict(np.transpose(representations))
# Update the counts.
counts[:, c] = np.bincount(predictions,
minlength=factor_counts[i])
mean_fairness[i, j], max_fairness[i,
j] = inter_group_fairness(counts)
# Report the scores.
scores.update(compute_scores_dict(mean_fairness, "mean_fairness"))
scores.update(compute_scores_dict(max_fairness, "max_fairness"))
return scores