def get_named_ground_truth_data(name): """Returns ground truth data set based on name. Args: name: String with the name of the dataset. Raises: ValueError: if an invalid data set name is provided. """ if name == "dsprites_full": return dsprites.DSprites([1, 2, 3, 4, 5]) elif name == "dsprites_noshape": return dsprites.DSprites([2, 3, 4, 5]) elif name == "color_dsprites": return dsprites.ColorDSprites([1, 2, 3, 4, 5]) elif name == "noisy_dsprites": return dsprites.NoisyDSprites([1, 2, 3, 4, 5]) elif name == "scream_dsprites": return dsprites.ScreamDSprites([1, 2, 3, 4, 5]) elif name == "smallnorb": return norb.SmallNORB() elif name == "cars3d": return cars3d.Cars3D() elif name == "shapes3d": return shapes3d.Shapes3D() elif name == "dummy_data": return dummy_data.DummyData() else: raise ValueError("Invalid data set name.")
def test_metric_lasso(self): ground_truth_data = dummy_data.DummyData() random_state = np.random.RandomState(0) num_factors = ground_truth_data.num_factors batch_size = 10 num_data_points = 1000 permutation = np.random.permutation(num_factors) sign_inverse = np.random.choice(num_factors, int(num_factors / 2)) def rep_fn1(data): return (np.reshape(data, (batch_size, -1))[:, :num_factors], np.ones(num_factors)) # Should be invariant to permutation and sign inverse. def rep_fn2(data): raw_representation = np.reshape(data, (batch_size, -1))[:, :num_factors] perm_rep = raw_representation[:, permutation] perm_rep[:, sign_inverse] = -1.0 * perm_rep[:, sign_inverse] return perm_rep, np.ones(num_factors) scores = udr.compute_udr_sklearn(ground_truth_data, [rep_fn1, rep_fn2], random_state, batch_size, num_data_points, correlation_matrix="lasso") self.assertBetween(scores["model_scores"][0], 0.8, 1.0) self.assertBetween(scores["model_scores"][1], 0.8, 1.0)
def test_metric_kl(self): ground_truth_data = dummy_data.DummyData() random_state = np.random.RandomState(0) num_factors = ground_truth_data.num_factors batch_size = 10 num_data_points = 1000 # Representation without KL Mask where only first latent is valid. def rep_fn(data): rep = np.concatenate([ np.reshape(data, (batch_size, -1))[:, :1], np.random.random_sample((batch_size, num_factors - 1)) ], axis=1) kl_mask = np.zeros(num_factors) kl_mask[0] = 1.0 return rep, kl_mask scores = udr.compute_udr_sklearn(ground_truth_data, [rep_fn, rep_fn], random_state, batch_size, num_data_points, filter_low_kl=False) self.assertBetween(scores["model_scores"][0], 0.0, 0.2) self.assertBetween(scores["model_scores"][1], 0.0, 0.2) scores = udr.compute_udr_sklearn(ground_truth_data, [rep_fn, rep_fn], random_state, batch_size, num_data_points, filter_low_kl=True) self.assertBetween(scores["model_scores"][0], 0.8, 1.0) self.assertBetween(scores["model_scores"][1], 0.8, 1.0)
def test_perfect_labeller(self, labels, target): ground_truth_data = dummy_data.DummyData() processed_labels, _ = semi_supervised_utils.perfect_labeller( labels, ground_truth_data, np.random.RandomState(0)) test_value = np.sum(np.abs(processed_labels - labels)) self.assertEqual(test_value, target)
def test_tfdata(self): ground_truth_data = dummy_data.DummyData() dataset = util.tf_data_set_from_ground_truth_data(ground_truth_data, 0) one_shot_iterator = dataset.make_one_shot_iterator() next_element = one_shot_iterator.get_next() with self.test_session() as sess: for _ in range(10): sess.run(next_element)
def test_metric(self): gin.bind_parameter("predictor.predictor_fn", utils.gradient_boosting_classifier) ground_truth_data = dummy_data.DummyData() def representation_function(x): return np.array(x, dtype=np.float64)[:, :, 0, 0] random_state = np.random.RandomState(0) _ = fairness.compute_fairness(ground_truth_data, representation_function, random_state, None, 1000, 1000)
def test_noisy_labeller(self, labels, target_low, target_high): ground_truth_data = dummy_data.DummyData() old_labels = labels.copy() processed_labels, _ = semi_supervised_utils.noisy_labeller( labels, ground_truth_data, np.random.RandomState(0), 0.1) index_equal = (processed_labels - old_labels).flatten() test_value = np.count_nonzero(index_equal) self.assertBetween(test_value, target_low, target_high)
def get_named_ground_truth_data(name, corr_type='plane', corr_indices=[3, 4], col=None): """Returns ground truth data set based on name. Args: name: String with the name of the dataset. Raises: ValueError: if an invalid data set name is provided. """ if name == "dsprites_full": return dsprites.DSprites([1, 2, 3, 4, 5]) elif name == "correlated_dsprites_full": return dsprites.CorrelatedDSprites([1, 2, 3, 4, 5], corr_indices, corr_type) elif name == "dsprites_noshape": return dsprites.DSprites([2, 3, 4, 5]) elif name == "correlated_dsprites_noshape": return dsprites.CorrelatedDSprites([2, 3, 4, 5], corr_indices, corr_type) elif name == "color_dsprites": return dsprites.ColorDSprites([1, 2, 3, 4, 5]) elif name == "backgroundcolor_dsprites": return dsprites.BackgroundColorDSprites([1, 2, 3, 4, 5, 6], col) elif name == "correlated_backgroundcolor_dsprites": return dsprites.CorrelatedBackgroundColorDSprites([1, 2, 3, 4, 5, 6], corr_indices, corr_type) elif name == "correlated_color_dsprites": return dsprites.CorrelatedColorDSprites([1, 2, 3, 4, 5], corr_indices, corr_type) elif name == "noisy_dsprites": return dsprites.NoisyDSprites([1, 2, 3, 4, 5]) elif name == "correlated_noisy_dsprites": return dsprites.CorrelatedNoisyDSprites([1, 2, 3, 4, 5], corr_indices, corr_type) elif name == "scream_dsprites": return dsprites.ScreamDSprites([1, 2, 3, 4, 5]) elif name == "correlated_scream_dsprites": return dsprites.CorrelatedScreamDSprites([1, 2, 3, 4, 5], corr_indices, corr_type) elif name == "smallnorb": return norb.SmallNORB() elif name == "cars3d": return cars3d.Cars3D() elif name == "shapes3d": return shapes3d.Shapes3D() elif name == "dummy_data": return dummy_data.DummyData() else: raise ValueError("Invalid data set name: " + name + ".")
def test_bin_labeller(self, labels, target, num_bins): labels = labels.reshape((1, 10)) target = target.reshape((1, 10)) ground_truth_data = dummy_data.DummyData() processed_labels, _ = semi_supervised_utils.bin_labeller( labels, ground_truth_data, np.random.RandomState(0), num_bins=num_bins) test_value = np.all(processed_labels == target) self.assertEqual(test_value, True)
def test_weak_data(self): ground_truth_data = dummy_data.DummyData() binding = ["dynamics.k = 1"] gin.parse_config_files_and_bindings([], binding) dataset = \ train_weak_lib.weak_dataset_from_ground_truth_data( ground_truth_data, 0) one_shot_iterator = tf.compat.v1.data.make_one_shot_iterator(dataset) next_element = one_shot_iterator.get_next() with self.test_session() as sess: elem = sess.run(next_element) self.assertEqual(elem[0].shape, (128, 64, 1)) self.assertEqual(elem[1].shape, (1, ))
def test_task(self): gin.bind_parameter("predictor.predictor_fn", utils.gradient_boosting_classifier) gin.bind_parameter("strong_downstream_task.num_train", [1000]) gin.bind_parameter("strong_downstream_task.num_test", 1000) gin.bind_parameter("strong_downstream_task.n_experiment", 2) ground_truth_data = dummy_data.DummyData() def representation_function(x): return np.array(x, dtype=np.float64)[:, :, 0, 0] random_state = np.random.RandomState(0) scores = strong_downstream_task.compute_strong_downstream_task( ground_truth_data, representation_function, random_state, artifact_dir=None) self.assertBetween(scores["1000:mean_strong_test_accuracy"], 0.0, 0.3)
def test_intervene(self): ground_truth_data = dummy_data.DummyData() random_state = np.random.RandomState(0) ys_train = ground_truth_data.sample_factors(1000, random_state) ys_test = ground_truth_data.sample_factors(1000, random_state) num_factors = ys_train.shape[1] for i in range(num_factors): (y_train_int, y_test_int, interv_factor, factor_interv_train) = strong_downstream_task.intervene( ys_train.copy(), ys_test.copy(), i, num_factors, ground_truth_data) assert interv_factor != i, "Wrong factor interevened on." assert (y_train_int[:, interv_factor] == factor_interv_train ).all(), "Training set not intervened on." assert (y_test_int[:, interv_factor] != factor_interv_train ).all(), "Training set not intervened on."
def get_named_ground_truth_data(name): """Returns ground truth data set based on name. Args: name: String with the name of the dataset. Raises: ValueError: if an invalid data set name is provided. """ if name == "dsprites_full": return dsprites.DSprites([1, 2, 3, 4, 5]) elif name == "dsprites_noshape": return dsprites.DSprites([2, 3, 4, 5]) elif name == "color_dsprites": return dsprites.ColorDSprites([1, 2, 3, 4, 5]) elif name == "noisy_dsprites": return dsprites.NoisyDSprites([1, 2, 3, 4, 5]) elif name == "scream_dsprites": return dsprites.ScreamDSprites([1, 2, 3, 4, 5]) elif name == "smallnorb": return norb.SmallNORB() elif name == "cars3d": return cars3d.Cars3D() elif name == "mpi3d_toy": return mpi3d.MPI3D(mode="mpi3d_toy") elif name == "mpi3d_realistic": return mpi3d.MPI3D(mode="mpi3d_realistic") elif name == "mpi3d_real": return mpi3d.MPI3D(mode="mpi3d_real") elif name == "shapes3d": return shapes3d.Shapes3D() elif name == "dummy_data": return dummy_data.DummyData() elif name == "modelnet": return DisLibGroundTruthData(**MODELNET_PARAMS) elif name == "arrow": return DisLibGroundTruthData(**ARROW_PARAMS) elif name == "pixel4": return DisLibGroundTruthData(**WRAPPED_PIXEL4_PARAMS) elif name == "pixel8": return DisLibGroundTruthData(**WRAPPED_PIXEL8_PARAMS) else: raise ValueError("Invalid data set name.")
def get_named_ground_truth_data(name): """Returns ground truth data set based on name. Args: name: String with the name of the dataset. Raises: ValueError: if an invalid data set name is provided. """ if name == "threeDotsCache": # a large random sample from ThreeDots return threeDots.ThreeDotsTrainingCache() elif name == "threeDots": return threeDots.ThreeDots() elif name == "dsprites_full": return dsprites.DSprites([1, 2, 3, 4, 5]) elif name == "dsprites_noshape": return dsprites.DSprites([2, 3, 4, 5]) elif name == "color_dsprites": return dsprites.ColorDSprites([1, 2, 3, 4, 5]) elif name == "noisy_dsprites": return dsprites.NoisyDSprites([1, 2, 3, 4, 5]) elif name == "scream_dsprites": return dsprites.ScreamDSprites([1, 2, 3, 4, 5]) elif name == "smallnorb": return norb.SmallNORB() elif name == "cars3d": return cars3d.Cars3D() elif name == "mpi3d_toy": return mpi3d.MPI3D(mode="mpi3d_toy") elif name == "mpi3d_realistic": return mpi3d.MPI3D(mode="mpi3d_realistic") elif name == "mpi3d_real": return mpi3d.MPI3D(mode="mpi3d_real") elif name == "mpi3d_multi_real": return mpi3d_multi.MPI3DMulti(mode="mpi3d_real") elif name == "shapes3d": return shapes3d.Shapes3D() elif name == "dummy_data": return dummy_data.DummyData() else: raise ValueError("Invalid data set name.")
def get_named_ground_truth_data(name): """Returns ground truth data set based on name. Args: name: String with the name of the dataset. Raises: ValueError: if an invalid data set name is provided. """ if name == "dsprites_full": return dsprites.DSprites([1, 2, 3, 4, 5]) elif name == "dsprites_noshape": return dsprites.DSprites([2, 3, 4, 5]) elif name == "color_dsprites": return dsprites.ColorDSprites([1, 2, 3, 4, 5]) elif name == "noisy_dsprites": return dsprites.NoisyDSprites([1, 2, 3, 4, 5]) elif name == "scream_dsprites": return dsprites.ScreamDSprites([1, 2, 3, 4, 5]) elif name == "smallnorb": return norb.SmallNORB() elif name == "cars3d": return cars3d.Cars3D() elif name == "mpi3d_toy": return mpi3d.MPI3D(mode="mpi3d_toy") elif name == "mpi3d_realistic": return mpi3d.MPI3D(mode="mpi3d_realistic") elif name == "mpi3d_real": return mpi3d.MPI3D(mode="mpi3d_real") elif name == "3dshapes": return shapes3d.Shapes3D() elif name == "3dshapes_holdout" or name == "3dshapes_pca_holdout_s5000": return shapes3d_partial.Shapes3DPartial(name) elif name == "3dshapes_model_all": return shapes3d_partial.Shapes3DPartial(name), None elif name[:8] == "3dshapes": return shapes3d_partial.Shapes3DPartial( name + '_train'), shapes3d_partial.Shapes3DPartial(name + '_valid') elif name == "dummy_data": return dummy_data.DummyData() else: raise ValueError("Invalid data set name.")
def test_semi_supervised_data(self): num_labels = 1000 gin.clear_config() gin_bindings = ["labeller.labeller_fn = @perfect_labeller"] gin.parse_config_files_and_bindings([], gin_bindings) ground_truth_data = dummy_data.DummyData() (sampled_observations, sampled_factors, _) = semi_supervised_utils.sample_supervised_data( 0, ground_truth_data, num_labels) dataset = train_s2_lib.semi_supervised_dataset_from_ground_truth_data( ground_truth_data, num_labels, 0, sampled_observations, sampled_factors) one_shot_iterator = dataset.make_one_shot_iterator() next_element = one_shot_iterator.get_next() with self.test_session() as sess: for _ in range(1): elem = sess.run(next_element) self.assertEqual(elem[0].shape, (64, 64, 1)) self.assertEqual(elem[1][0].shape, (64, 64, 1)) self.assertLen(elem[1][1], 10)