def test_generate_local_batch_factor_code(self):
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = lambda x: np.array(x, dtype=np.float64)
num_points = 100
random_state = np.random.RandomState(3)
# you gotta test batch size smaller than num_points, silly
batch_size = 13
local_repr, local_facts = utils.generate_local_batch_factor_code(
ground_truth_data,
representation_function,
num_points,
random_state,
batch_size,
locality_proportion=1.0,
continuity_cutoff=0.0)
for local_batch in [local_repr, local_facts]:
np.testing.assert_equal(local_batch.shape, [10, num_points])
for inx in range(10):
self.assertEqual(np.min(local_batch[inx, :]), 0)
self.assertEqual(np.max(local_batch[inx, :]), 10 - 1)
local_repr, local_facts = utils.generate_local_batch_factor_code(
ground_truth_data,
representation_function,
num_points,
random_state,
batch_size,
locality_proportion=0.1,
continuity_cutoff=0.0)
# representation is identity
for local_batch in [local_repr, local_facts]:
np.testing.assert_equal(local_batch.shape, [10, num_points])
for inx in range(10):
assert_correct_radius(self, local_batch[inx, :], 1, 0, 10 - 1)
def test_metric(self):
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = lambda x: np.array(x, dtype=np.float64)
random_state = np.random.RandomState(0)
scores = dci.compute_dci(
ground_truth_data, representation_function, random_state, 1000, 1000)
self.assertBetween(scores["disentanglement"], 0.9, 1.0)
self.assertBetween(scores["completeness"], 0.9, 1.0)
def test_metric(self):
initialize_gin()
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = lambda x: np.array(x, dtype=np.float64)
random_state = np.random.RandomState(0)
scores = local_modularity.compute_local_modularity(
ground_truth_data, representation_function, random_state)
self.assertBetween(scores["modularity_score"], 0.9, 1.0)
def test_bad_metric(self):
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = lambda x: np.zeros_like(x, dtype=np.float64)
random_state = np.random.RandomState(0)
scores = sap_score.compute_sap(
ground_truth_data, representation_function, random_state, None, 3000,
3000, continuous_factors=True)
self.assertBetween(scores["SAP_score"], 0.0, 0.2)
def test_metric(self):
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = lambda x: x
random_state = np.random.RandomState(0)
scores = beta_vae.compute_beta_vae_sklearn(ground_truth_data,
representation_function,
random_state, 5, 2000, 2000)
self.assertBetween(scores["train_accuracy"], 0.9, 1.0)
self.assertBetween(scores["eval_accuracy"], 0.9, 1.0)
def test_bad_metric(self):
gin.bind_parameter("discretizer.discretizer_fn", _identity_discretizer)
gin.bind_parameter("discretizer.num_bins", 10)
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = np.zeros_like
random_state = np.random.RandomState(0)
scores = mig.compute_mig(
ground_truth_data, representation_function, random_state, None, 3000)
self.assertBetween(scores["discrete_mig"], 0.0, 0.2)
def test_bad_metric(self):
initialize_gin()
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = np.zeros_like
random_state = np.random.RandomState(0)
scores = local_mig.compute_local_mig(ground_truth_data,
representation_function,
random_state)
self.assertBetween(scores["discrete_mig"], 0.0, 0.2)
def test_duplicated_latent_space(self):
ground_truth_data = dummy_data.IdentityObservationsData()
def representation_function(x):
x = np.array(x, dtype=np.float64)
return np.hstack([x, x])
random_state = np.random.RandomState(0)
scores = sap_score.compute_sap(
ground_truth_data, representation_function, random_state, None, 3000,
3000, continuous_factors=True)
self.assertBetween(scores["SAP_score"], 0.0, 0.2)
def test_bad_metric(self):
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = np.zeros_like
random_state = np.random.RandomState(0)
scores = factor_vae.compute_factor_vae(ground_truth_data,
representation_function,
random_state, 5, 3000, 2000,
2500)
self.assertBetween(scores["train_accuracy"], 0.0, 0.2)
self.assertBetween(scores["eval_accuracy"], 0.0, 0.2)
def test_bad_metric(self):
gin.bind_parameter("discretizer.discretizer_fn", _identity_discretizer)
gin.bind_parameter("discretizer.num_bins", 10)
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = lambda x: np.zeros_like(x, dtype=np.float64)
random_state = np.random.RandomState(0)
scores = irs.compute_irs(ground_truth_data, representation_function,
random_state, 0.99, 3000, 3000)
self.assertBetween(scores["IRS"], 0.0, 0.1)
def test_metric(self):
gin.bind_parameter("discretizer.discretizer_fn", _identity_discretizer)
gin.bind_parameter("discretizer.num_bins", 10)
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = lambda x: np.array(x, dtype=np.float64)
random_state = np.random.RandomState(0)
scores = modularity_explicitness.compute_modularity_explicitness(
ground_truth_data, representation_function, random_state, None,
3000, 3000)
self.assertBetween(scores["modularity_score"], 0.9, 1.0)
def test_duplicated_latent_space(self):
ground_truth_data = dummy_data.IdentityObservationsData()
def representation_function(x):
x = np.array(x, dtype=np.float64)
return np.hstack([x, x])
random_state = np.random.RandomState(0)
scores = dci.compute_dci(
ground_truth_data, representation_function, random_state, 1000, 1000)
self.assertBetween(scores["disentanglement"], 0.9, 1.0)
target = 1. - np.log(2)/np.log(10)
self.assertBetween(scores["completeness"], target-.1, target+.1)
def test_duplicated_latent_space(self):
gin.bind_parameter("discretizer.discretizer_fn", _identity_discretizer)
gin.bind_parameter("discretizer.num_bins", 10)
ground_truth_data = dummy_data.IdentityObservationsData()
def representation_function(x):
x = np.array(x, dtype=np.float64)
return np.hstack([x, x])
random_state = np.random.RandomState(0)
scores = mig.compute_mig(
ground_truth_data, representation_function, random_state, None, 3000)
self.assertBetween(scores["discrete_mig"], 0.0, 0.1)
def test_metric_dci(self):
gin.bind_parameter("discretizer.discretizer_fn", _identity_discretizer)
gin.bind_parameter("discretizer.num_bins", 10)
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = lambda x: np.array(x, dtype=np.float64)
random_state = np.random.RandomState(0)
scores = unified_scores.compute_unified_scores(
ground_truth_data, representation_function, random_state, None, 10000,
100, matrix_fns=[unified_scores.importance_gbt_matrix])
self.assertBetween(
scores["importance_gbt_matrix.dci_disentanglement"], 0.9, 1.0)
def test_duplicated_latent_space(self):
initialize_gin()
ground_truth_data = dummy_data.IdentityObservationsData()
def representation_function(x):
x = np.array(x, dtype=np.float64)
return np.hstack([x, x])
random_state = np.random.RandomState(0)
scores = local_modularity.compute_local_modularity(
ground_truth_data, representation_function, random_state)
self.assertBetween(scores["modularity_score"], 0.9, 1.0)
def test_duplicated_latent_space_sap(self):
gin.bind_parameter("discretizer.discretizer_fn", _identity_discretizer)
gin.bind_parameter("discretizer.num_bins", 10)
ground_truth_data = dummy_data.IdentityObservationsData()
def representation_function(x):
x = np.array(x, dtype=np.float64)
return np.hstack([x, x])
random_state = np.random.RandomState(0)
scores = unified_scores.compute_unified_scores(
ground_truth_data, representation_function, random_state, None, 1000,
1000, matrix_fns=[unified_scores.accuracy_svm_matrix])
self.assertBetween(scores["accuracy_svm_matrix.sap"], 0.0, 0.2)
def test_generate_batch_factor_code(self):
ground_truth_data = dummy_data.IdentityObservationsData()
representation_function = lambda x: np.array(x, dtype=np.float64)
num_points = 100
random_state = np.random.RandomState(3)
batch_size = 192
represents, factors = utils.generate_batch_factor_code(
ground_truth_data, representation_function, num_points,
random_state, batch_size)
# representation is identity
for batch in [represents, factors]:
np.testing.assert_equal(batch.shape, [10, num_points])
for inx in range(10):
self.assertEqual(np.min(batch[inx, :]), 0)
self.assertEqual(np.max(batch[inx, :]), 10 - 1)
def test_bad_metric(self):
ground_truth_data = dummy_data.IdentityObservationsData()
random_state_rep = np.random.RandomState(0)
# The representation which randomly permutes the factors, should have equal
# non-zero importance which should give a low modularity score.
def representation_function(x):
code = np.array(x, dtype=np.float64)
for i in range(code.shape[0]):
code[i, :] = random_state_rep.permutation(code[i, :])
return code
random_state = np.random.RandomState(0)
scores = dci.compute_dci(
ground_truth_data, representation_function, random_state, 1000, 1000)
self.assertBetween(scores["disentanglement"], 0.0, 0.2)
self.assertBetween(scores["completeness"], 0.0, 0.2)
def test_bad_metric(self):
initialize_gin()
ground_truth_data = dummy_data.IdentityObservationsData()
random_state_rep = np.random.RandomState(0)
# The representation which randomly permutes the factors, should have equal
# non-zero MI which should give a low modularity score.
def representation_function(x):
code = np.array(x, dtype=np.float64)
for i in range(code.shape[0]):
code[i, :] = random_state_rep.permutation(code[i, :])
return code
random_state = np.random.RandomState(0)
scores = local_modularity.compute_local_modularity(
ground_truth_data, representation_function, random_state)
self.assertBetween(scores["modularity_score"], 0.0, 0.2)
def test_bad_metric(self):
gin.bind_parameter("discretizer.discretizer_fn", _identity_discretizer)
gin.bind_parameter("discretizer.num_bins", 10)
ground_truth_data = dummy_data.IdentityObservationsData()
random_state_rep = np.random.RandomState(0)
# The representation which randomly permutes the factors, should have equal
# non-zero MI which should give a low modularity score.
def representation_function(x):
code = np.array(x, dtype=np.float64)
for i in range(code.shape[0]):
code[i, :] = random_state_rep.permutation(code[i, :])
return code
random_state = np.random.RandomState(0)
scores = modularity_explicitness.compute_modularity_explicitness(
ground_truth_data, representation_function, random_state, None,
20000, 20000)
self.assertBetween(scores["modularity_score"], 0.0, 0.2)
