def _compute_mig(mus_train, ys_train):
"""Computes score based on both training and testing codes and factors."""
score_dict = {}
discretized_mus = utils.make_discretizer(mus_train)
m = utils.discrete_mutual_info(discretized_mus, ys_train)
assert m.shape[0] == mus_train.shape[0]
assert m.shape[1] == ys_train.shape[0]
# m is [num_latents, num_factors]
entropy = utils.discrete_entropy(ys_train)
sorted_m = np.sort(m, axis=0)[::-1]
score_dict["discrete_mig"] = np.mean(
np.divide(sorted_m[0, :] - sorted_m[1, :], entropy[:]))
return score_dict
def unsupervised_metrics(ground_truth_data,
representation_function,
random_state,
artifact_dir=None,
num_train=gin.REQUIRED,
batch_size=16):
"""Computes unsupervised scores based on covariance and mutual information.
Args:
ground_truth_data: GroundTruthData to be sampled from.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
random_state: Numpy random state used for randomness.
artifact_dir: Optional path to directory where artifacts can be saved.
num_train: Number of points used for training.
batch_size: Batch size for sampling.
Returns:
Dictionary with scores.
"""
del artifact_dir
scores = {}
logging.info("Generating training set.")
mus_train, _ = utils.generate_batch_factor_code(ground_truth_data,
representation_function,
num_train, random_state,
batch_size)
num_codes = mus_train.shape[0]
cov_mus = np.cov(mus_train)
assert num_codes == cov_mus.shape[0]
# Gaussian total correlation.
scores["gaussian_total_correlation"] = gaussian_total_correlation(cov_mus)
# Gaussian Wasserstein correlation.
scores[
"gaussian_wasserstein_correlation"] = gaussian_wasserstein_correlation(
cov_mus)
scores["gaussian_wasserstein_correlation_norm"] = (
scores["gaussian_wasserstein_correlation"] / np.sum(np.diag(cov_mus)))
# Compute average mutual information between different factors.
mus_discrete = utils.make_discretizer(mus_train)
mutual_info_matrix = utils.discrete_mutual_info(mus_discrete, mus_discrete)
np.fill_diagonal(mutual_info_matrix, 0)
mutual_info_score = np.sum(mutual_info_matrix) / (num_codes**2 - num_codes)
scores["mutual_info_score"] = mutual_info_score
return scores
def compute_modularity_explicitness(ground_truth_data,
representation_function,
random_state,
artifact_dir=None,
num_train=gin.REQUIRED,
num_test=gin.REQUIRED,
batch_size=16):
"""Computes the modularity metric according to Sec 3.
Args:
ground_truth_data: GroundTruthData to be sampled from.
representation_function: Function that takes observations as input and
outputs a dim_representation sized representation for each observation.
random_state: Numpy random state used for randomness.
artifact_dir: Optional path to directory where artifacts can be saved.
num_train: Number of points used for training.
num_test: Number of points used for testing.
batch_size: Batch size for sampling.
Returns:
Dictionary with average modularity score and average explicitness
(train and test).
"""
del artifact_dir
scores = {}
mus_train, ys_train = utils.generate_batch_factor_code(
ground_truth_data, representation_function, num_train, random_state,
batch_size)
mus_test, ys_test = utils.generate_batch_factor_code(
ground_truth_data, representation_function, num_test, random_state,
batch_size)
discretized_mus = utils.make_discretizer(mus_train)
mutual_information = utils.discrete_mutual_info(discretized_mus, ys_train)
# Mutual information should have shape [num_codes, num_factors].
assert mutual_information.shape[0] == mus_train.shape[0]
assert mutual_information.shape[1] == ys_train.shape[0]
scores["modularity_score"] = modularity(mutual_information)
explicitness_score_train = np.zeros([ys_train.shape[0], 1])
explicitness_score_test = np.zeros([ys_test.shape[0], 1])
mus_train_norm, mean_mus, stddev_mus = utils.normalize_data(mus_train)
mus_test_norm, _, _ = utils.normalize_data(mus_test, mean_mus, stddev_mus)
for i in range(ys_train.shape[0]):
explicitness_score_train[i], explicitness_score_test[i] = \
explicitness_per_factor(mus_train_norm, ys_train[i, :],
mus_test_norm, ys_test[i, :])
scores["explicitness_score_train"] = np.mean(explicitness_score_train)
scores["explicitness_score_test"] = np.mean(explicitness_score_test)
return scores
def test_discrete_mutual_info(self):
xs = np.array([[1, 2, 1, 2], [1, 1, 2, 2]])
ys = np.array([[1, 2, 1, 2], [2, 2, 1, 1]])
result = utils.discrete_mutual_info(xs, ys)
shouldbe = np.array([[np.log(2), 0.], [0., np.log(2)]])
np.testing.assert_allclose(result, shouldbe)