def _compute_mig(mus_train, ys_train): """Computes score based on both training and testing codes and factors.""" score_dict = {} discretized_mus = utils.make_discretizer(mus_train) m = utils.discrete_mutual_info(discretized_mus, ys_train) assert m.shape[0] == mus_train.shape[0] assert m.shape[1] == ys_train.shape[0] # m is [num_latents, num_factors] entropy = utils.discrete_entropy(ys_train) sorted_m = np.sort(m, axis=0)[::-1] score_dict["discrete_mig"] = np.mean( np.divide(sorted_m[0, :] - sorted_m[1, :], entropy[:])) return score_dict
def unsupervised_metrics(ground_truth_data, representation_function, random_state, artifact_dir=None, num_train=gin.REQUIRED, batch_size=16): """Computes unsupervised scores based on covariance and mutual information. Args: ground_truth_data: GroundTruthData to be sampled from. representation_function: Function that takes observations as input and outputs a dim_representation sized representation for each observation. random_state: Numpy random state used for randomness. artifact_dir: Optional path to directory where artifacts can be saved. num_train: Number of points used for training. batch_size: Batch size for sampling. Returns: Dictionary with scores. """ del artifact_dir scores = {} logging.info("Generating training set.") mus_train, _ = utils.generate_batch_factor_code(ground_truth_data, representation_function, num_train, random_state, batch_size) num_codes = mus_train.shape[0] cov_mus = np.cov(mus_train) assert num_codes == cov_mus.shape[0] # Gaussian total correlation. scores["gaussian_total_correlation"] = gaussian_total_correlation(cov_mus) # Gaussian Wasserstein correlation. scores[ "gaussian_wasserstein_correlation"] = gaussian_wasserstein_correlation( cov_mus) scores["gaussian_wasserstein_correlation_norm"] = ( scores["gaussian_wasserstein_correlation"] / np.sum(np.diag(cov_mus))) # Compute average mutual information between different factors. mus_discrete = utils.make_discretizer(mus_train) mutual_info_matrix = utils.discrete_mutual_info(mus_discrete, mus_discrete) np.fill_diagonal(mutual_info_matrix, 0) mutual_info_score = np.sum(mutual_info_matrix) / (num_codes**2 - num_codes) scores["mutual_info_score"] = mutual_info_score return scores
def compute_modularity_explicitness(ground_truth_data, representation_function, random_state, artifact_dir=None, num_train=gin.REQUIRED, num_test=gin.REQUIRED, batch_size=16): """Computes the modularity metric according to Sec 3. Args: ground_truth_data: GroundTruthData to be sampled from. representation_function: Function that takes observations as input and outputs a dim_representation sized representation for each observation. random_state: Numpy random state used for randomness. artifact_dir: Optional path to directory where artifacts can be saved. num_train: Number of points used for training. num_test: Number of points used for testing. batch_size: Batch size for sampling. Returns: Dictionary with average modularity score and average explicitness (train and test). """ del artifact_dir scores = {} mus_train, ys_train = utils.generate_batch_factor_code( ground_truth_data, representation_function, num_train, random_state, batch_size) mus_test, ys_test = utils.generate_batch_factor_code( ground_truth_data, representation_function, num_test, random_state, batch_size) discretized_mus = utils.make_discretizer(mus_train) mutual_information = utils.discrete_mutual_info(discretized_mus, ys_train) # Mutual information should have shape [num_codes, num_factors]. assert mutual_information.shape[0] == mus_train.shape[0] assert mutual_information.shape[1] == ys_train.shape[0] scores["modularity_score"] = modularity(mutual_information) explicitness_score_train = np.zeros([ys_train.shape[0], 1]) explicitness_score_test = np.zeros([ys_test.shape[0], 1]) mus_train_norm, mean_mus, stddev_mus = utils.normalize_data(mus_train) mus_test_norm, _, _ = utils.normalize_data(mus_test, mean_mus, stddev_mus) for i in range(ys_train.shape[0]): explicitness_score_train[i], explicitness_score_test[i] = \ explicitness_per_factor(mus_train_norm, ys_train[i, :], mus_test_norm, ys_test[i, :]) scores["explicitness_score_train"] = np.mean(explicitness_score_train) scores["explicitness_score_test"] = np.mean(explicitness_score_test) return scores
def compute_irs(ground_truth_data, representation_function, random_state, artifact_dir=None, diff_quantile=0.99, num_train=gin.REQUIRED, batch_size=gin.REQUIRED): """Computes the Interventional Robustness Score. Args: ground_truth_data: GroundTruthData to be sampled from. representation_function: Function that takes observations as input and outputs a dim_representation sized representation for each observation. random_state: Numpy random state used for randomness. artifact_dir: Optional path to directory where artifacts can be saved. diff_quantile: Float value between 0 and 1 to decide what quantile of diffs to select (use 1.0 for the version in the paper). num_train: Number of points used for training. batch_size: Batch size for sampling. Returns: Dict with IRS and number of active dimensions. """ del artifact_dir logging.info("Generating training set.") mus, ys = utils.generate_batch_factor_code(ground_truth_data, representation_function, num_train, random_state, batch_size) assert mus.shape[1] == num_train ys_discrete = utils.make_discretizer(ys) active_mus = _drop_constant_dims(mus) if not active_mus.any(): irs_score = 0.0 else: irs_score = scalable_disentanglement_score(ys_discrete.T, active_mus.T, diff_quantile)["avg_score"] score_dict = {} score_dict["IRS"] = irs_score score_dict["num_active_dims"] = np.sum(active_mus) return score_dict