def compute_downstream_task(ground_truth_data,
representation_function,
random_state,
artifact_dir=None,
num_train=gin.REQUIRED,
num_test=gin.REQUIRED,
batch_size=16):
"""Computes loss of downstream task.
Args:
ground_truth_data: GroundTruthData to be sampled from.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
random_state: Numpy random state used for randomness.
artifact_dir: Optional path to directory where artifacts can be saved.
num_train: Number of points used for training.
num_test: Number of points used for testing.
batch_size: Batch size for sampling.
Returns:
Dictionary with scores.
"""
del artifact_dir
scores = {}
for train_size in num_train:
mus_train, ys_train = utils.generate_batch_factor_code(
ground_truth_data, representation_function, train_size,
random_state, batch_size)
mus_test, ys_test = utils.generate_batch_factor_code(
ground_truth_data, representation_function, num_test, random_state,
batch_size)
predictor_model = utils.make_predictor_fn()
print(mus_train.shape, ys_train.shape)
train_err, test_err = _compute_loss(np.transpose(mus_train), ys_train,
np.transpose(mus_test), ys_test,
predictor_model)
size_string = str(train_size)
scores[size_string + ":mean_train_accuracy"] = np.mean(train_err)
scores[size_string + ":mean_test_accuracy"] = np.mean(test_err)
scores[size_string + ":min_train_accuracy"] = np.min(train_err)
scores[size_string + ":min_test_accuracy"] = np.min(test_err)
for i in range(len(train_err)):
scores[size_string +
":train_accuracy_factor_{}".format(i)] = train_err[i]
scores[size_string +
":test_accuracy_factor_{}".format(i)] = test_err[i]
return scores
def compute_fairness(ground_truth_data,
representation_function,
random_state,
artifact_dir=None,
num_train=gin.REQUIRED,
num_test_points_per_class=gin.REQUIRED,
batch_size=16):
"""Computes unfairness scores.
We first compute either the mean or maximum total variation for a given
sensitive and target variable. Then, we either average or take the maximum
with respect to target and sensitive variable. For convenience, we compute and
save all combinations. The score used in Section 4 of the paper is here called
mean_fairness:mean_pred:mean_sens.
Args:
ground_truth_data: GroundTruthData to be sampled from.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
random_state: Numpy random state used for randomness.
artifact_dir: Optional path to directory where artifacts can be saved.
num_train: Number of points used for training.
num_test_points_per_class: Number of points used for testing.
batch_size: Batch size for sampling.
Returns:
Dictionary with scores.
"""
del artifact_dir
factor_counts = ground_truth_data.factors_num_values
num_factors = len(factor_counts)
scores = {}
# Training a predictive model.
mus_train, ys_train = utils.generate_batch_factor_code(
ground_truth_data, representation_function, num_train, random_state,
batch_size)
predictor_model_fn = utils.make_predictor_fn()
# For each factor train a single predictive model.
mean_fairness = np.zeros((num_factors, num_factors), dtype=np.float64)
max_fairness = np.zeros((num_factors, num_factors), dtype=np.float64)
for i in range(num_factors):
model = predictor_model_fn()
model.fit(np.transpose(mus_train), ys_train[i, :])
for j in range(num_factors):
if i == j:
continue
# Sample a random set of factors once.
original_factors = ground_truth_data.sample_factors(
num_test_points_per_class, random_state)
counts = np.zeros((factor_counts[i], factor_counts[j]),
dtype=np.int64)
for c in range(factor_counts[j]):
# Intervene on the sensitive attribute.
intervened_factors = np.copy(original_factors)
intervened_factors[:, j] = c
# Obtain the batched observations.
observations = ground_truth_data.sample_observations_from_factors(
intervened_factors, random_state)
representations = utils.obtain_representation(
observations, representation_function, batch_size)
# Get the predictions.
predictions = model.predict(np.transpose(representations))
# Update the counts.
counts[:, c] = np.bincount(predictions,
minlength=factor_counts[i])
mean_fairness[i, j], max_fairness[i,
j] = inter_group_fairness(counts)
# Report the scores.
scores.update(compute_scores_dict(mean_fairness, "mean_fairness"))
scores.update(compute_scores_dict(max_fairness, "max_fairness"))
return scores
def compute_strong_downstream_task(ground_truth_data,
representation_function,
random_state,
artifact_dir=None,
num_train=gin.REQUIRED,
num_test=gin.REQUIRED,
n_experiment=gin.REQUIRED):
"""Computes loss of downstream task.
This task is about strong generalization under covariate shifts. We first
perform an intervention fixing a value for a factor in the whole training set.
Then, we train a GBT classifier, and at test time, we consider all other
values for that factor. We repeat the experiment n_experiment times, to ensure
robustness.
Args:
ground_truth_data: GroundTruthData to be sampled from.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
random_state: Numpy random state used for randomness.
artifact_dir: Optional path to directory where artifacts can be saved.
num_train: Number of points used for training.
num_test: Number of points used for testing.
n_experiment: Number of repetitions of the experiment.
Returns:
Dictionary with scores.
"""
del artifact_dir
scores = {}
for train_size in num_train:
# sample factors
factors_train = ground_truth_data.sample_factors(train_size, random_state)
factors_test = ground_truth_data.sample_factors(num_test, random_state)
# obtain_observations without intervention
x_train = ground_truth_data.sample_observations_from_factors(
factors_train, random_state)
x_test = ground_truth_data.sample_observations_from_factors(
factors_test, random_state)
mus_train = representation_function(x_train)
mus_test = representation_function(x_test)
# train predictor on data without interbention
predictor_model = utils.make_predictor_fn()
y_train = np.transpose(factors_train)
y_test = np.transpose(factors_test)
train_err, test_err = _compute_loss(
mus_train, y_train, mus_test,
y_test, predictor_model)
# train predictor on data with interventions
train_err_int, test_err_int = _compute_loss_intervene(
factors_train, factors_test, predictor_model, ground_truth_data,
representation_function, random_state, n_experiment)
size_string = str(train_size)
scores[size_string +
":mean_train_accuracy"] = np.mean(train_err)
scores[size_string +
":mean_test_accuracy"] = np.mean(test_err)
scores[size_string +
":mean_strong_train_accuracy"] = np.mean(train_err_int)
scores[size_string +
":mean_strong_test_accuracy"] = np.mean(test_err_int)
scores[size_string + ":strong_generalization_gap"] = 1. - (
scores[size_string + ":mean_strong_test_accuracy"] /
scores[size_string + ":mean_test_accuracy"])
return scores
def compute_downstream_regression_on_representations(
ground_truth_data,
representation_function,
random_state,
holdout_dataset_name=gin.REQUIRED,
artifact_dir=None,
num_train=gin.REQUIRED,
num_test=gin.REQUIRED,
num_holdout=gin.REQUIRED,
batch_size=16):
"""Computes loss of downstream task on representations.
Args:
ground_truth_data: GroundTruthData to be sampled from.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
random_state: Numpy random state used for randomness.
artifact_dir: Optional path to directory where artifacts can be saved.
num_train: Number of points used for training.
num_test: Number of points used for testing.
batch_size: Batch size for sampling.
Returns:
Dictionary with scores.
"""
del artifact_dir
ground_truth_train_data, ground_truth_test_data = ground_truth_data
ground_truth_holdout_data = named_data.get_named_ground_truth_data(
holdout_dataset_name)
scores = {}
for train_size in num_train:
mus_train, ys_train = utils.generate_batch_label_code(
ground_truth_train_data, representation_function, train_size,
random_state, batch_size)
mus_test, ys_test = utils.generate_batch_label_code(
ground_truth_test_data, representation_function, num_test,
random_state, batch_size)
mus_holdout, ys_holdout = utils.generate_batch_label_code(
ground_truth_holdout_data, representation_function, num_holdout,
random_state, batch_size)
predictor_model = utils.make_predictor_fn()
train_err, test_err, holdout_err, random_normal_err, random_uniform_err = \
_compute_mse_loss(
np.transpose(mus_train), ys_train, np.transpose(mus_test),
ys_test, np.transpose(mus_holdout), ys_holdout, predictor_model)
size_string = str(train_size)
scores[size_string + ":mean_train_mse"] = np.mean(train_err)
scores[size_string + ":mean_test_mse"] = np.mean(test_err)
scores[size_string + ":mean_holdout_mse"] = np.mean(holdout_err)
scores[size_string +
":mean_random_normal_mse"] = np.mean(random_normal_err)
scores[size_string +
":mean_random_uniform_mse"] = np.mean(random_uniform_err)
scores[size_string + ":min_train_mse"] = np.min(train_err)
scores[size_string + ":min_test_mse"] = np.min(test_err)
scores[size_string + ":min_holdout_mse"] = np.min(holdout_err)
scores[size_string +
":min_random_normal_mse"] = np.min(random_normal_err)
scores[size_string +
":min_random_uniform_mse"] = np.min(random_uniform_err)
for i in range(len(train_err)):
scores[size_string +
":train_mse_factor_{}".format(i)] = train_err[i]
scores[size_string + ":test_mse_factor_{}".format(i)] = test_err[i]
scores[size_string +
":holdout_mse_factor_{}".format(i)] = holdout_err[i]
scores[size_string + ":random_normal_mse_factor_{}".format(
i)] = random_normal_err[i]
scores[size_string + ":random_uniform_mse_factor_{}".format(
i)] = random_uniform_err[i]
return scores
def compute_reduced_downstream_task(ground_truth_data,
representation_function,
random_state,
artifact_dir=None,
num_factors_to_remove=gin.REQUIRED,
num_train=gin.REQUIRED,
num_test=gin.REQUIRED,
batch_size=16):
"""Computes loss of a reduced downstream task.
Measure the information leakage in each latent component after removing the
k ("factors_to_remove") most informative features for the prediction task.
Args:
ground_truth_data: GroundTruthData to be sampled from.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
random_state: Numpy random state used for randomness.
artifact_dir: Optional path to directory where artifacts can be saved.
num_factors_to_remove: Number of factors to remove from the latent
representation.
num_train: Number of points used for training.
num_test: Number of points used for testing.
batch_size: Batch size for sampling.
Returns:
Dictionary with scores.
"""
del artifact_dir
scores = {}
# Loop on different sizes of the training 'batch', as specified with gin.
for train_size in num_train:
size_string = str(train_size)
mus_train, ys_train = utils.generate_batch_factor_code(
ground_truth_data, representation_function, train_size, random_state,
batch_size)
mus_test, ys_test = utils.generate_batch_factor_code(
ground_truth_data, representation_function, num_test, random_state,
batch_size)
# Create variables for aggregated scores.
reduced_factor_train_scores = []
other_factors_train_scores = []
reduced_factor_test_scores = []
other_factors_test_scores = []
# Compute the reduced representation and test it for each factor of
# variation.
for factor_of_interest in range(ground_truth_data.num_factors):
# Copy the training data and eliminate the k most informative factors.
reduced_mus_train = mus_train.copy()
reduced_mus_test = mus_test.copy()
for _ in range(num_factors_to_remove):
reduced_mus_train, reduced_mus_test =\
compute_reduced_representation(reduced_mus_train, ys_train,
reduced_mus_test, ys_test,
factor_of_interest)
predictor_model = utils.make_predictor_fn()
train_acc, test_acc = compute_predictive_accuracy(
np.transpose(reduced_mus_train), ys_train,
np.transpose(reduced_mus_test), ys_test, predictor_model)
# Save scores for reduced factor.
scores[size_string +
":reduced_factor_{}:mean_train_accuracy_reduced_factor".format(
factor_of_interest)] = train_acc[factor_of_interest]
scores[size_string +
":reduced_factor_{}:mean_test_accuracy_reduced_factor".format(
factor_of_interest)] = test_acc[factor_of_interest]
reduced_factor_train_scores.append(train_acc[factor_of_interest])
reduced_factor_test_scores.append(test_acc[factor_of_interest])
# Save the scores (accuracies) in the score dictionary.
local_other_factors_train_scores = []
local_other_factors_test_scores = []
for i in range(len(train_acc)):
scores[size_string +
":reduced_factor_{}:mean_train_accuracy_factor_{}".format(
factor_of_interest, i)] = train_acc[i]
scores[size_string +
":reduced_factor_{}:mean_test_accuracy_factor_{}".format(
factor_of_interest, i)] = test_acc[i]
if i != factor_of_interest:
local_other_factors_train_scores.append(train_acc[i])
local_other_factors_test_scores.append(test_acc[i])
# Save mean score for non-reduced factors.
scores[size_string +
":reduced_factor_{}:mean_train_accuracy_non_reduced_factor".format(
factor_of_interest)] = np.mean(
local_other_factors_train_scores)
scores[size_string +
":reduced_factor_{}:mean_test_accuracy_non_reduced_factor".format(
factor_of_interest)] = np.mean(local_other_factors_test_scores)
other_factors_train_scores.append(
np.mean(local_other_factors_train_scores))
other_factors_test_scores.append(np.mean(local_other_factors_test_scores))
# Compute the aggregate scores.
scores[size_string + ":mean_train_accuracy_reduced_factor"] = np.mean(
reduced_factor_train_scores)
scores[size_string + ":mean_test_accuracy_reduced_factor"] = np.mean(
reduced_factor_test_scores)
scores[size_string + ":mean_train_accuracy_other_factors"] = np.mean(
other_factors_train_scores)
scores[size_string + ":mean_test_accuracy_other_factors"] = np.mean(
other_factors_test_scores)
return scores