def _prepare_rng_critical_parts(seed=1234): torch.random.manual_seed(seed) initial_benchmark = PluginTests.create_benchmark(seed=seed) val_benchmark = benchmark_with_validation_stream( initial_benchmark, 0.3, shuffle=True ) return (val_benchmark, _PlainMLP(input_size=6, hidden_size=10))
def test_lazy_benchmark_with_validation_stream_fixed_size(self): lazy_options = [None, True, False] for lazy_option in lazy_options: with self.subTest(lazy_option=lazy_option): pattern_shape = (3, 32, 32) # Definition of training experiences # Experience 1 experience_1_x = torch.zeros(100, *pattern_shape) experience_1_y = torch.zeros(100, dtype=torch.long) experience_1_dataset = AvalancheTensorDataset( experience_1_x, experience_1_y) # Experience 2 experience_2_x = torch.zeros(80, *pattern_shape) experience_2_y = torch.ones(80, dtype=torch.long) experience_2_dataset = AvalancheTensorDataset( experience_2_x, experience_2_y) # Test experience test_x = torch.zeros(50, *pattern_shape) test_y = torch.zeros(50, dtype=torch.long) experience_test = AvalancheTensorDataset(test_x, test_y) def train_gen(): # Lazy generator of the training stream for dataset in [ experience_1_dataset, experience_2_dataset ]: yield dataset def test_gen(): # Lazy generator of the test stream for dataset in [experience_test]: yield dataset initial_benchmark_instance = create_lazy_generic_benchmark( train_generator=LazyStreamDefinition( train_gen(), 2, [0, 0]), test_generator=LazyStreamDefinition(test_gen(), 1, [0]), complete_test_set_only=True, dataset_type=AvalancheDatasetType.CLASSIFICATION, ) valid_benchmark = benchmark_with_validation_stream( initial_benchmark_instance, 20, shuffle=False, lazy_splitting=lazy_option, ) if lazy_option is None or lazy_option: expect_laziness = True else: expect_laziness = False self.assertEqual( expect_laziness, valid_benchmark.stream_definitions["train"].is_lazy, ) self.assertEqual(2, len(valid_benchmark.train_stream)) self.assertEqual(2, len(valid_benchmark.valid_stream)) self.assertEqual(1, len(valid_benchmark.test_stream)) self.assertTrue(valid_benchmark.complete_test_set_only) maybe_exp = valid_benchmark.stream_definitions[ "train"].exps_data.get_experience_if_loaded(0) self.assertEqual(expect_laziness, maybe_exp is None) self.assertEqual(80, len(valid_benchmark.train_stream[0].dataset)) maybe_exp = valid_benchmark.stream_definitions[ "train"].exps_data.get_experience_if_loaded(1) self.assertEqual(expect_laziness, maybe_exp is None) self.assertEqual(60, len(valid_benchmark.train_stream[1].dataset)) maybe_exp = valid_benchmark.stream_definitions[ "valid"].exps_data.get_experience_if_loaded(0) self.assertEqual(expect_laziness, maybe_exp is None) self.assertEqual(20, len(valid_benchmark.valid_stream[0].dataset)) maybe_exp = valid_benchmark.stream_definitions[ "valid"].exps_data.get_experience_if_loaded(1) self.assertEqual(expect_laziness, maybe_exp is None) self.assertEqual(20, len(valid_benchmark.valid_stream[1].dataset)) self.assertIsNotNone( valid_benchmark.stream_definitions["train"].exps_data. get_experience_if_loaded(0)) self.assertIsNotNone( valid_benchmark.stream_definitions["valid"].exps_data. get_experience_if_loaded(0)) self.assertIsNotNone( valid_benchmark.stream_definitions["train"].exps_data. get_experience_if_loaded(1)) self.assertIsNotNone( valid_benchmark.stream_definitions["valid"].exps_data. get_experience_if_loaded(1)) self.assertTrue( torch.equal( experience_1_x[:80], valid_benchmark.train_stream[0].dataset[:][0], )) self.assertTrue( torch.equal( experience_2_x[:60], valid_benchmark.train_stream[1].dataset[:][0], )) self.assertTrue( torch.equal( experience_1_y[:80], valid_benchmark.train_stream[0].dataset[:][1], )) self.assertTrue( torch.equal( experience_2_y[:60], valid_benchmark.train_stream[1].dataset[:][1], )) self.assertTrue( torch.equal( experience_1_x[80:], valid_benchmark.valid_stream[0].dataset[:][0], )) self.assertTrue( torch.equal( experience_2_x[60:], valid_benchmark.valid_stream[1].dataset[:][0], )) self.assertTrue( torch.equal( experience_1_y[80:], valid_benchmark.valid_stream[0].dataset[:][1], )) self.assertTrue( torch.equal( experience_2_y[60:], valid_benchmark.valid_stream[1].dataset[:][1], )) self.assertTrue( torch.equal(test_x, valid_benchmark.test_stream[0].dataset[:][0])) self.assertTrue( torch.equal(test_y, valid_benchmark.test_stream[0].dataset[:][1]))
def test_benchmark_with_validation_stream_fixed_size(self): pattern_shape = (3, 32, 32) # Definition of training experiences # Experience 1 experience_1_x = torch.zeros(100, *pattern_shape) experience_1_y = torch.zeros(100, dtype=torch.long) # Experience 2 experience_2_x = torch.zeros(80, *pattern_shape) experience_2_y = torch.ones(80, dtype=torch.long) # Test experience test_x = torch.zeros(50, *pattern_shape) test_y = torch.zeros(50, dtype=torch.long) initial_benchmark_instance = tensors_benchmark( train_tensors=[(experience_1_x, experience_1_y), (experience_2_x, experience_2_y)], test_tensors=[(test_x, test_y)], task_labels=[0, 0], # Task label of each train exp complete_test_set_only=True) valid_benchmark = benchmark_with_validation_stream( initial_benchmark_instance, 20, shuffle=False) self.assertEqual(2, len(valid_benchmark.train_stream)) self.assertEqual(2, len(valid_benchmark.valid_stream)) self.assertEqual(1, len(valid_benchmark.test_stream)) self.assertTrue(valid_benchmark.complete_test_set_only) self.assertEqual(80, len(valid_benchmark.train_stream[0].dataset)) self.assertEqual(60, len(valid_benchmark.train_stream[1].dataset)) self.assertEqual(20, len(valid_benchmark.valid_stream[0].dataset)) self.assertEqual(20, len(valid_benchmark.valid_stream[1].dataset)) self.assertTrue( torch.equal(experience_1_x[:80], valid_benchmark.train_stream[0].dataset[:][0])) self.assertTrue( torch.equal(experience_2_x[:60], valid_benchmark.train_stream[1].dataset[:][0])) self.assertTrue( torch.equal(experience_1_y[:80], valid_benchmark.train_stream[0].dataset[:][1])) self.assertTrue( torch.equal(experience_2_y[:60], valid_benchmark.train_stream[1].dataset[:][1])) self.assertTrue( torch.equal(experience_1_x[80:], valid_benchmark.valid_stream[0].dataset[:][0])) self.assertTrue( torch.equal(experience_2_x[60:], valid_benchmark.valid_stream[1].dataset[:][0])) self.assertTrue( torch.equal(experience_1_y[80:], valid_benchmark.valid_stream[0].dataset[:][1])) self.assertTrue( torch.equal(experience_2_y[60:], valid_benchmark.valid_stream[1].dataset[:][1])) self.assertTrue( torch.equal(test_x, valid_benchmark.test_stream[0].dataset[:][0])) self.assertTrue( torch.equal(test_y, valid_benchmark.test_stream[0].dataset[:][1]))
def test_benchmark_with_validation_stream_rel_size(self): pattern_shape = (3, 32, 32) # Definition of training experiences # Experience 1 experience_1_x = torch.zeros(100, *pattern_shape) experience_1_y = torch.zeros(100, dtype=torch.long) # Experience 2 experience_2_x = torch.zeros(80, *pattern_shape) experience_2_y = torch.ones(80, dtype=torch.long) # Test experience test_x = torch.zeros(50, *pattern_shape) test_y = torch.zeros(50, dtype=torch.long) initial_benchmark_instance = tensors_benchmark( train_tensors=[ (experience_1_x, experience_1_y), (experience_2_x, experience_2_y), ], test_tensors=[(test_x, test_y)], task_labels=[0, 0], # Task label of each train exp complete_test_set_only=True, ) valid_benchmark = benchmark_with_validation_stream( initial_benchmark_instance, 0.2, shuffle=False) expected_rel_1_valid = int(100 * 0.2) expected_rel_1_train = 100 - expected_rel_1_valid expected_rel_2_valid = int(80 * 0.2) expected_rel_2_train = 80 - expected_rel_2_valid self.assertEqual(2, len(valid_benchmark.train_stream)) self.assertEqual(2, len(valid_benchmark.valid_stream)) self.assertEqual(1, len(valid_benchmark.test_stream)) self.assertTrue(valid_benchmark.complete_test_set_only) self.assertEqual(expected_rel_1_train, len(valid_benchmark.train_stream[0].dataset)) self.assertEqual(expected_rel_2_train, len(valid_benchmark.train_stream[1].dataset)) self.assertEqual(expected_rel_1_valid, len(valid_benchmark.valid_stream[0].dataset)) self.assertEqual(expected_rel_2_valid, len(valid_benchmark.valid_stream[1].dataset)) self.assertTrue( torch.equal( experience_1_x[:expected_rel_1_train], valid_benchmark.train_stream[0].dataset[:][0], )) self.assertTrue( torch.equal( experience_2_x[:expected_rel_2_train], valid_benchmark.train_stream[1].dataset[:][0], )) self.assertTrue( torch.equal( experience_1_y[:expected_rel_1_train], valid_benchmark.train_stream[0].dataset[:][1], )) self.assertTrue( torch.equal( experience_2_y[:expected_rel_2_train], valid_benchmark.train_stream[1].dataset[:][1], )) self.assertTrue( torch.equal( experience_1_x[expected_rel_1_train:], valid_benchmark.valid_stream[0].dataset[:][0], )) self.assertTrue( torch.equal( experience_2_x[expected_rel_2_train:], valid_benchmark.valid_stream[1].dataset[:][0], )) self.assertTrue( torch.equal( experience_1_y[expected_rel_1_train:], valid_benchmark.valid_stream[0].dataset[:][1], )) self.assertTrue( torch.equal( experience_2_y[expected_rel_2_train:], valid_benchmark.valid_stream[1].dataset[:][1], )) self.assertTrue( torch.equal(test_x, valid_benchmark.test_stream[0].dataset[:][0])) self.assertTrue( torch.equal(test_y, valid_benchmark.test_stream[0].dataset[:][1]))
def test_lazy_benchmark_with_validation_stream_fixed_size(self): pattern_shape = (3, 32, 32) # Definition of training experiences # Experience 1 experience_1_x = torch.zeros(100, *pattern_shape) experience_1_y = torch.zeros(100, dtype=torch.long) experience_1_dataset = AvalancheTensorDataset(experience_1_x, experience_1_y) # Experience 2 experience_2_x = torch.zeros(80, *pattern_shape) experience_2_y = torch.ones(80, dtype=torch.long) experience_2_dataset = AvalancheTensorDataset(experience_2_x, experience_2_y) # Test experience test_x = torch.zeros(50, *pattern_shape) test_y = torch.zeros(50, dtype=torch.long) experience_test = AvalancheTensorDataset(test_x, test_y) def train_gen(): # Lazy generator of the training stream for dataset in [experience_1_dataset, experience_2_dataset]: yield dataset def test_gen(): # Lazy generator of the test stream for dataset in [experience_test]: yield dataset initial_benchmark_instance = create_lazy_generic_benchmark( train_generator=LazyStreamDefinition(train_gen(), 2, [0, 0]), test_generator=LazyStreamDefinition(test_gen(), 1, [0]), complete_test_set_only=True, dataset_type=AvalancheDatasetType.CLASSIFICATION) valid_benchmark = benchmark_with_validation_stream( initial_benchmark_instance, 20, shuffle=False) self.assertEqual(2, len(valid_benchmark.train_stream)) self.assertEqual(2, len(valid_benchmark.valid_stream)) self.assertEqual(1, len(valid_benchmark.test_stream)) self.assertTrue(valid_benchmark.complete_test_set_only) self.assertEqual(80, len(valid_benchmark.train_stream[0].dataset)) self.assertEqual(60, len(valid_benchmark.train_stream[1].dataset)) self.assertEqual(20, len(valid_benchmark.valid_stream[0].dataset)) self.assertEqual(20, len(valid_benchmark.valid_stream[1].dataset)) self.assertTrue( torch.equal(experience_1_x[:80], valid_benchmark.train_stream[0].dataset[:][0])) self.assertTrue( torch.equal(experience_2_x[:60], valid_benchmark.train_stream[1].dataset[:][0])) self.assertTrue( torch.equal(experience_1_y[:80], valid_benchmark.train_stream[0].dataset[:][1])) self.assertTrue( torch.equal(experience_2_y[:60], valid_benchmark.train_stream[1].dataset[:][1])) self.assertTrue( torch.equal(experience_1_x[80:], valid_benchmark.valid_stream[0].dataset[:][0])) self.assertTrue( torch.equal(experience_2_x[60:], valid_benchmark.valid_stream[1].dataset[:][0])) self.assertTrue( torch.equal(experience_1_y[80:], valid_benchmark.valid_stream[0].dataset[:][1])) self.assertTrue( torch.equal(experience_2_y[60:], valid_benchmark.valid_stream[1].dataset[:][1])) self.assertTrue( torch.equal(test_x, valid_benchmark.test_stream[0].dataset[:][0])) self.assertTrue( torch.equal(test_y, valid_benchmark.test_stream[0].dataset[:][1]))