def test_complete_test_set_only(self): train_exps = [] test_exps = [] for _ in range(5): tensor_x = torch.rand(200, 3, 28, 28) tensor_y = torch.randint(0, 100, (200, )) tensor_t = torch.randint(0, 5, (200, )) train_exps.append( AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t)) for _ in range(3): tensor_x = torch.rand(150, 3, 28, 28) tensor_y = torch.randint(0, 100, (150, )) tensor_t = torch.randint(0, 5, (150, )) test_exps.append( AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t)) with self.assertRaises(Exception): benchmark_instance = GenericCLScenario( stream_definitions={ "train": (train_exps, ), "test": (test_exps, ), }, complete_test_set_only=True, ) benchmark_instance = GenericCLScenario( stream_definitions={ "train": (train_exps, ), "test": (test_exps[0], ), }, complete_test_set_only=True, ) self.assertEqual(5, len(benchmark_instance.train_stream)) self.assertEqual(1, len(benchmark_instance.test_stream))
def test_custom_streams_name_and_length(self): train_exps = [] test_exps = [] valid_exps = [] for _ in range(5): tensor_x = torch.rand(200, 3, 28, 28) tensor_y = torch.randint(0, 100, (200,)) tensor_t = torch.randint(0, 5, (200,)) train_exps.append( AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t) ) for _ in range(3): tensor_x = torch.rand(150, 3, 28, 28) tensor_y = torch.randint(0, 100, (150,)) tensor_t = torch.randint(0, 3, (150,)) test_exps.append( AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t) ) for _ in range(4): tensor_x = torch.rand(220, 3, 28, 28) tensor_y = torch.randint(0, 100, (220,)) tensor_t = torch.randint(0, 5, (220,)) valid_exps.append( AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t) ) valid_origin_dataset = AvalancheTensorDataset( torch.ones(10, 3, 32, 32), torch.zeros(10) ) valid_t_labels = [{9}, {4, 5}, {7, 8}, {0}, {3}] with self.assertRaises(Exception): benchmark_instance = GenericCLScenario( stream_definitions={ "train": (train_exps,), "test": (test_exps,), "valid": (valid_exps, valid_t_labels, valid_origin_dataset), } ) valid_t_labels = valid_t_labels[:-1] benchmark_instance = GenericCLScenario( stream_definitions={ "train": (train_exps,), "test": (test_exps,), "valid": (valid_exps, valid_t_labels, valid_origin_dataset), } ) self.assertEqual(5, len(benchmark_instance.train_stream)) self.assertEqual(3, len(benchmark_instance.test_stream)) self.assertEqual(4, len(benchmark_instance.valid_stream)) self.assertEqual(None, benchmark_instance.original_train_dataset) self.assertEqual(None, benchmark_instance.original_test_dataset) self.assertEqual( valid_origin_dataset, benchmark_instance.original_valid_dataset ) for i, exp in enumerate(benchmark_instance.train_stream): expect_x, expect_y, expect_t = train_exps[i][0] got_x, got_y, got_t = exp.dataset[0] self.assertTrue(torch.equal(expect_x, got_x)) self.assertTrue(torch.equal(expect_y, got_y)) self.assertEqual(int(expect_t), got_t) exp_t_labels = set(exp.task_labels) self.assertLess(max(exp_t_labels), 5) self.assertGreaterEqual(min(exp_t_labels), 0) for i, exp in enumerate(benchmark_instance.test_stream): expect_x, expect_y, expect_t = test_exps[i][0] got_x, got_y, got_t = exp.dataset[0] self.assertTrue(torch.equal(expect_x, got_x)) self.assertTrue(torch.equal(expect_y, got_y)) self.assertEqual(int(expect_t), got_t) exp_t_labels = set(exp.task_labels) self.assertLess(max(exp_t_labels), 3) self.assertGreaterEqual(min(exp_t_labels), 0) for i, exp in enumerate(benchmark_instance.valid_stream): expect_x, expect_y, expect_t = valid_exps[i][0] got_x, got_y, got_t = exp.dataset[0] self.assertTrue(torch.equal(expect_x, got_x)) self.assertTrue(torch.equal(expect_y, got_y)) self.assertEqual(int(expect_t), got_t) exp_t_labels = set(exp.task_labels) self.assertEqual(valid_t_labels[i], exp_t_labels)
def test_lazy_scenario_drop_old_ones(self): train_exps, test_exps, other_stream_exps = self._make_tensor_datasets() train_dataset_exp_0_weak_ref = weakref.ref(train_exps[0]) train_dataset_exp_1_weak_ref = weakref.ref(train_exps[1]) train_gen = GenericCLScenarioTests._generate_stream(train_exps) test_gen = GenericCLScenarioTests._generate_stream(test_exps) other_gen = GenericCLScenarioTests._generate_stream(other_stream_exps) benchmark_instance = GenericCLScenario(stream_definitions=dict( train=((train_gen, len(train_exps)), [ train_exps[0].targets_task_labels, train_exps[1].targets_task_labels ]), test=((test_gen, len(test_exps)), [test_exps[0].targets_task_labels]), other=((other_gen, len(other_stream_exps)), [other_stream_exps[0].targets_task_labels]))) # --- START: Test classes timeline before first experience --- current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertIsNone(current_classes) self.assertSetEqual(set(), set(prev_classes)) self.assertIsNone(cumulative_classes) self.assertIsNone(future_classes) # --- END: Test classes timeline before first experience --- train_exp_0: GenericExperience = benchmark_instance.train_stream[0] # --- START: Test classes timeline at first experience --- current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertSetEqual(set(train_exps[0].targets), set(current_classes)) self.assertSetEqual(set(), set(prev_classes)) self.assertSetEqual(set(train_exps[0].targets), set(cumulative_classes)) self.assertIsNone(future_classes) current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(1) self.assertIsNone(current_classes) self.assertSetEqual(set(train_exps[0].targets), set(prev_classes)) # None because we didn't load exp 0 yet self.assertIsNone(cumulative_classes) self.assertSetEqual(set(), set(future_classes)) # --- END: Test classes timeline at first experience --- # Check if it works when the previous experience is dropped benchmark_instance.train_stream.drop_previous_experiences(0) train_exp_1: GenericExperience = benchmark_instance.train_stream[1] # --- START: Test classes timeline at second experience --- # Check if get_classes_timeline(0) is consistent current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertSetEqual(set(train_exps[0].targets), set(current_classes)) self.assertSetEqual(set(), set(prev_classes)) self.assertSetEqual(set(train_exps[0].targets), set(cumulative_classes)) # We now have access to future classes! self.assertSetEqual(set(train_exps[1].targets), set(future_classes)) current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(1) self.assertSetEqual(set(train_exps[1].targets), set(current_classes)) self.assertSetEqual(set(train_exps[0].targets), set(prev_classes)) self.assertSetEqual( set(train_exps[0].targets).union(set(train_exps[1].targets)), set(cumulative_classes)) self.assertSetEqual(set(), set(future_classes)) # --- END: Test classes timeline at second experience --- train_0_classes = train_exp_0.classes_in_this_experience train_1_classes = train_exp_1.classes_in_this_experience train_0_classes_min = min(train_0_classes) train_1_classes_min = min(train_1_classes) train_0_classes_max = max(train_0_classes) train_1_classes_max = max(train_1_classes) self.assertGreaterEqual(train_0_classes_min, 0) self.assertLess(train_0_classes_max, 70) self.assertGreaterEqual(train_1_classes_min, 0) self.assertLess(train_1_classes_max, 100) with self.assertRaises(IndexError): train_exp_2: GenericExperience = benchmark_instance.train_stream[2] test_exp_0: GenericExperience = benchmark_instance.test_stream[0] test_0_classes = test_exp_0.classes_in_this_experience test_0_classes_min = min(test_0_classes) test_0_classes_max = max(test_0_classes) self.assertGreaterEqual(test_0_classes_min, 100) self.assertLess(test_0_classes_max, 200) with self.assertRaises(IndexError): test_exp_1: GenericExperience = benchmark_instance.test_stream[1] other_exp_0: GenericExperience = benchmark_instance.other_stream[0] other_0_classes = other_exp_0.classes_in_this_experience other_0_classes_min = min(other_0_classes) other_0_classes_max = max(other_0_classes) self.assertGreaterEqual(other_0_classes_min, 400) self.assertLess(other_0_classes_max, 600) with self.assertRaises(IndexError): other_exp_1: GenericExperience = benchmark_instance.other_stream[1] train_exps = None train_exp_0 = None train_exp_1 = None train_0_classes = None train_1_classes = None train_gen = None # The generational GC is needed, ref-count is not enough here gc.collect() # This will check that the train dataset of exp0 has been garbage # collected correctly self.assertIsNone(train_dataset_exp_0_weak_ref()) self.assertIsNotNone(train_dataset_exp_1_weak_ref()) benchmark_instance.train_stream.drop_previous_experiences(1) gc.collect() # This will check that exp1 has been garbage collected correctly self.assertIsNone(train_dataset_exp_0_weak_ref()) self.assertIsNone(train_dataset_exp_1_weak_ref()) with self.assertRaises(Exception): exp_0 = benchmark_instance.train_stream[0] with self.assertRaises(Exception): exp_1 = benchmark_instance.train_stream[1]
def test_lazy_scenario(self): train_exps, test_exps, other_stream_exps = self._make_tensor_datasets() def train_gen(): # Lazy generator of the training stream for dataset in train_exps: yield dataset def test_gen(): # Lazy generator of the test stream for dataset in test_exps: yield dataset def other_gen(): # Lazy generator of the "other" stream for dataset in other_stream_exps: yield dataset benchmark_instance = GenericCLScenario(stream_definitions=dict( train=((train_gen(), len(train_exps)), [ train_exps[0].targets_task_labels, train_exps[1].targets_task_labels ]), test=((test_gen(), len(test_exps)), [test_exps[0].targets_task_labels]), other=((other_gen(), len(other_stream_exps)), [other_stream_exps[0].targets_task_labels]))) # --- START: Test classes timeline before first experience --- current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertIsNone(current_classes) self.assertSetEqual(set(), set(prev_classes)) self.assertIsNone(cumulative_classes) self.assertIsNone(future_classes) # --- END: Test classes timeline before first experience --- train_exp_0: GenericExperience = benchmark_instance.train_stream[0] # --- START: Test classes timeline at first experience --- current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertSetEqual(set(train_exps[0].targets), set(current_classes)) self.assertSetEqual(set(), set(prev_classes)) self.assertSetEqual(set(train_exps[0].targets), set(cumulative_classes)) self.assertIsNone(future_classes) current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(1) self.assertIsNone(current_classes) self.assertSetEqual(set(train_exps[0].targets), set(prev_classes)) # None because we didn't load exp 0 yet self.assertIsNone(cumulative_classes) self.assertSetEqual(set(), set(future_classes)) # --- END: Test classes timeline at first experience --- train_exp_1: GenericExperience = benchmark_instance.train_stream[1] # --- START: Test classes timeline at second experience --- # Check if get_classes_timeline(0) is consistent current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertSetEqual(set(train_exps[0].targets), set(current_classes)) self.assertSetEqual(set(), set(prev_classes)) self.assertSetEqual(set(train_exps[0].targets), set(cumulative_classes)) # We now have access to future classes! self.assertSetEqual(set(train_exps[1].targets), set(future_classes)) current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(1) self.assertSetEqual(set(train_exps[1].targets), set(current_classes)) self.assertSetEqual(set(train_exps[0].targets), set(prev_classes)) self.assertSetEqual( set(train_exps[0].targets).union(set(train_exps[1].targets)), set(cumulative_classes)) self.assertSetEqual(set(), set(future_classes)) # --- END: Test classes timeline at second experience --- train_0_classes = train_exp_0.classes_in_this_experience train_1_classes = train_exp_1.classes_in_this_experience train_0_classes_min = min(train_0_classes) train_1_classes_min = min(train_1_classes) train_0_classes_max = max(train_0_classes) train_1_classes_max = max(train_1_classes) self.assertGreaterEqual(train_0_classes_min, 0) self.assertLess(train_0_classes_max, 70) self.assertGreaterEqual(train_1_classes_min, 0) self.assertLess(train_1_classes_max, 100) with self.assertRaises(IndexError): train_exp_2: GenericExperience = benchmark_instance.train_stream[2] test_exp_0: GenericExperience = benchmark_instance.test_stream[0] test_0_classes = test_exp_0.classes_in_this_experience test_0_classes_min = min(test_0_classes) test_0_classes_max = max(test_0_classes) self.assertGreaterEqual(test_0_classes_min, 100) self.assertLess(test_0_classes_max, 200) with self.assertRaises(IndexError): test_exp_1: GenericExperience = benchmark_instance.test_stream[1] other_exp_0: GenericExperience = benchmark_instance.other_stream[0] other_0_classes = other_exp_0.classes_in_this_experience other_0_classes_min = min(other_0_classes) other_0_classes_max = max(other_0_classes) self.assertGreaterEqual(other_0_classes_min, 400) self.assertLess(other_0_classes_max, 600) with self.assertRaises(IndexError): other_exp_1: GenericExperience = benchmark_instance.other_stream[1]
def benchmark_with_validation_stream( benchmark_instance: GenericCLScenario, validation_size: Union[int, float], shuffle: bool = False, input_stream: str = 'train', output_stream: str = 'valid', custom_split_strategy: Callable[[Experience], Tuple[AvalancheDataset, AvalancheDataset]] = None, *, experience_factory: Callable[[GenericScenarioStream, int], Experience] = None, lazy_splitting: bool = None): """ Helper that can be used to obtain a benchmark with a validation stream. This generator accepts an existing benchmark instance and returns a version of it in which a validation stream has been added. In its base form this generator will split train experiences to extract validation experiences of a fixed (by number of instances or relative size), configurable, size. The split can be also performed on other streams if needed and the name of the resulting validation stream can be configured too. Each validation experience will be extracted directly from a single training experience. Patterns selected for the validation experience will be removed from the training one. If shuffle is True, the validation stream will be created randomly. Beware that no kind of class balancing is done. The `custom_split_strategy` parameter can be used if a more specific splitting is required. Please note that the resulting experiences will have a task labels field equal to the one of the originating experience. Experience splitting can be executed in a lazy way. This behavior can be controlled using the `lazy_splitting` parameter. By default, experiences are split in a lazy way only when the input stream is lazily generated. :param benchmark_instance: The benchmark to split. :param validation_size: The size of the validation experience, as an int or a float between 0 and 1. Ignored if `custom_split_strategy` is used. :param shuffle: If True, patterns will be allocated to the validation stream randomly. This will use the default PyTorch random number generator at its current state. Defaults to False. Ignored if `custom_split_strategy` is used. If False, the first instances will be allocated to the training dataset by leaving the last ones to the validation dataset. :param input_stream: The name of the input stream. Defaults to 'train'. :param output_stream: The name of the output stream. Defaults to 'valid'. :param custom_split_strategy: A function that implements a custom splitting strategy. The function must accept an experience and return a tuple containing the new train and validation dataset. Defaults to None, which means that the standard splitting strategy will be used (which creates experiences according to `validation_size` and `shuffle`). A good starting to understand the mechanism is to look at the implementation of the standard splitting function :func:`random_validation_split_strategy`. :param experience_factory: The experience factory. Defaults to :class:`GenericExperience`. :param lazy_splitting: If True, the stream will be split in a lazy way. If False, the stream will be split immediately. Defaults to None, which means that the stream will be split in a lazy or non-lazy way depending on the laziness of the `input_stream`. :return: A benchmark instance in which the validation stream has been added. """ split_strategy = custom_split_strategy if split_strategy is None: split_strategy = partial( random_validation_split_strategy, validation_size, shuffle) stream_definitions: TStreamsUserDict = dict( benchmark_instance.stream_definitions) streams = benchmark_instance.streams if input_stream not in streams: raise ValueError(f'Stream {input_stream} could not be found in the ' f'benchmark instance') if output_stream in streams: raise ValueError(f'Stream {output_stream} already exists in the ' f'benchmark instance') stream = streams[input_stream] split_lazily = lazy_splitting if split_lazily is None: split_lazily = stream_definitions[input_stream].is_lazy exps_tasks_labels = list( stream_definitions[input_stream].exps_task_labels ) if not split_lazily: # Classic static splitting train_exps_source = [] valid_exps_source = [] exp: Experience for exp in stream: train_exp, valid_exp = split_strategy(exp) train_exps_source.append(train_exp) valid_exps_source.append(valid_exp) else: # Lazy splitting (based on a generator) split_generator = _lazy_train_val_split(split_strategy, stream) train_exps_gen, valid_exps_gen = _gen_split(split_generator) train_exps_source = (train_exps_gen, len(stream)) valid_exps_source = (valid_exps_gen, len(stream)) train_stream_def = \ StreamUserDef( train_exps_source, exps_tasks_labels, stream_definitions[input_stream].origin_dataset, split_lazily) valid_stream_def = \ StreamUserDef( valid_exps_source, exps_tasks_labels, stream_definitions[input_stream].origin_dataset, split_lazily) stream_definitions[input_stream] = train_stream_def stream_definitions[output_stream] = valid_stream_def complete_test_set_only = benchmark_instance.complete_test_set_only return GenericCLScenario(stream_definitions=stream_definitions, complete_test_set_only=complete_test_set_only, experience_factory=experience_factory)
def data_incremental_benchmark( benchmark_instance: GenericCLScenario, experience_size: int, shuffle: bool = False, drop_last: bool = False, split_streams: Sequence[str] = ('train',), custom_split_strategy: Callable[[Experience], Sequence[AvalancheDataset]] = None, experience_factory: Callable[[GenericScenarioStream, int], Experience] = None): """ High-level benchmark generator for a Data Incremental setup. This generator accepts an existing benchmark instance and returns a version of it in which experiences have been split in order to produce a Data Incremental stream. In its base form this generator will split train experiences in experiences of a fixed, configurable, size. The split can be also performed on other streams (like the test one) if needed. The `custom_split_strategy` parameter can be used if a more specific splitting is required. Beware that experience splitting is NOT executed in a lazy way. This means that the splitting process takes place immediately. Consider optimizing the split process for speed when using a custom splitting strategy. Please note that each mini-experience will have a task labels field equal to the one of the originating experience. The `complete_test_set_only` field of the resulting benchmark instance will be `True` only if the same field of original benchmark instance is `True` and if the resulting test stream contains exactly one experience. :param benchmark_instance: The benchmark to split. :param experience_size: The size of the experience, as an int. Ignored if `custom_split_strategy` is used. :param shuffle: If True, experiences will be split by first shuffling instances in each experience. This will use the default PyTorch random number generator at its current state. Defaults to False. Ignored if `custom_split_strategy` is used. :param drop_last: If True, if the last experience doesn't contain `experience_size` instances, then the last experience will be dropped. Defaults to False. Ignored if `custom_split_strategy` is used. :param split_streams: The list of streams to split. By default only the "train" stream will be split. :param custom_split_strategy: A function that implements a custom splitting strategy. The function must accept an experience and return a list of datasets each describing an experience. Defaults to None, which means that the standard splitting strategy will be used (which creates experiences of size `experience_size`). A good starting to understand the mechanism is to look at the implementation of the standard splitting function :func:`fixed_size_experience_split_strategy`. :param experience_factory: The experience factory. Defaults to :class:`GenericExperience`. :return: The Data Incremental benchmark instance. """ split_strategy = custom_split_strategy if split_strategy is None: split_strategy = partial( fixed_size_experience_split_strategy, experience_size, shuffle, drop_last) stream_definitions: TStreamsUserDict = dict( benchmark_instance.stream_definitions) for stream_name in split_streams: if stream_name not in stream_definitions: raise ValueError(f'Stream {stream_name} could not be found in the ' f'benchmark instance') stream = getattr(benchmark_instance, f'{stream_name}_stream') split_datasets: List[AvalancheDataset] = [] split_task_labels: List[Set[int]] = [] exp: Experience for exp in stream: experiences = split_strategy(exp) split_datasets += experiences for _ in range(len(experiences)): split_task_labels.append(set(exp.task_labels)) stream_def = StreamUserDef( split_datasets, split_task_labels, stream_definitions[stream_name].origin_dataset, False) stream_definitions[stream_name] = stream_def complete_test_set_only = benchmark_instance.complete_test_set_only and \ len(stream_definitions['test'].exps_data) == 1 return GenericCLScenario(stream_definitions=stream_definitions, complete_test_set_only=complete_test_set_only, experience_factory=experience_factory)