def test_lazy_scenario_drop_old_ones(self): train_exps, test_exps, other_stream_exps = self._make_tensor_datasets() train_dataset_exp_0_weak_ref = weakref.ref(train_exps[0]) train_dataset_exp_1_weak_ref = weakref.ref(train_exps[1]) train_gen = GenericCLScenarioTests._generate_stream(train_exps) test_gen = GenericCLScenarioTests._generate_stream(test_exps) other_gen = GenericCLScenarioTests._generate_stream(other_stream_exps) benchmark_instance = GenericCLScenario(stream_definitions=dict( train=((train_gen, len(train_exps)), [ train_exps[0].targets_task_labels, train_exps[1].targets_task_labels ]), test=((test_gen, len(test_exps)), [test_exps[0].targets_task_labels]), other=((other_gen, len(other_stream_exps)), [other_stream_exps[0].targets_task_labels]))) # --- START: Test classes timeline before first experience --- current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertIsNone(current_classes) self.assertSetEqual(set(), set(prev_classes)) self.assertIsNone(cumulative_classes) self.assertIsNone(future_classes) # --- END: Test classes timeline before first experience --- train_exp_0: GenericExperience = benchmark_instance.train_stream[0] # --- START: Test classes timeline at first experience --- current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertSetEqual(set(train_exps[0].targets), set(current_classes)) self.assertSetEqual(set(), set(prev_classes)) self.assertSetEqual(set(train_exps[0].targets), set(cumulative_classes)) self.assertIsNone(future_classes) current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(1) self.assertIsNone(current_classes) self.assertSetEqual(set(train_exps[0].targets), set(prev_classes)) # None because we didn't load exp 0 yet self.assertIsNone(cumulative_classes) self.assertSetEqual(set(), set(future_classes)) # --- END: Test classes timeline at first experience --- # Check if it works when the previous experience is dropped benchmark_instance.train_stream.drop_previous_experiences(0) train_exp_1: GenericExperience = benchmark_instance.train_stream[1] # --- START: Test classes timeline at second experience --- # Check if get_classes_timeline(0) is consistent current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertSetEqual(set(train_exps[0].targets), set(current_classes)) self.assertSetEqual(set(), set(prev_classes)) self.assertSetEqual(set(train_exps[0].targets), set(cumulative_classes)) # We now have access to future classes! self.assertSetEqual(set(train_exps[1].targets), set(future_classes)) current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(1) self.assertSetEqual(set(train_exps[1].targets), set(current_classes)) self.assertSetEqual(set(train_exps[0].targets), set(prev_classes)) self.assertSetEqual( set(train_exps[0].targets).union(set(train_exps[1].targets)), set(cumulative_classes)) self.assertSetEqual(set(), set(future_classes)) # --- END: Test classes timeline at second experience --- train_0_classes = train_exp_0.classes_in_this_experience train_1_classes = train_exp_1.classes_in_this_experience train_0_classes_min = min(train_0_classes) train_1_classes_min = min(train_1_classes) train_0_classes_max = max(train_0_classes) train_1_classes_max = max(train_1_classes) self.assertGreaterEqual(train_0_classes_min, 0) self.assertLess(train_0_classes_max, 70) self.assertGreaterEqual(train_1_classes_min, 0) self.assertLess(train_1_classes_max, 100) with self.assertRaises(IndexError): train_exp_2: GenericExperience = benchmark_instance.train_stream[2] test_exp_0: GenericExperience = benchmark_instance.test_stream[0] test_0_classes = test_exp_0.classes_in_this_experience test_0_classes_min = min(test_0_classes) test_0_classes_max = max(test_0_classes) self.assertGreaterEqual(test_0_classes_min, 100) self.assertLess(test_0_classes_max, 200) with self.assertRaises(IndexError): test_exp_1: GenericExperience = benchmark_instance.test_stream[1] other_exp_0: GenericExperience = benchmark_instance.other_stream[0] other_0_classes = other_exp_0.classes_in_this_experience other_0_classes_min = min(other_0_classes) other_0_classes_max = max(other_0_classes) self.assertGreaterEqual(other_0_classes_min, 400) self.assertLess(other_0_classes_max, 600) with self.assertRaises(IndexError): other_exp_1: GenericExperience = benchmark_instance.other_stream[1] train_exps = None train_exp_0 = None train_exp_1 = None train_0_classes = None train_1_classes = None train_gen = None # The generational GC is needed, ref-count is not enough here gc.collect() # This will check that the train dataset of exp0 has been garbage # collected correctly self.assertIsNone(train_dataset_exp_0_weak_ref()) self.assertIsNotNone(train_dataset_exp_1_weak_ref()) benchmark_instance.train_stream.drop_previous_experiences(1) gc.collect() # This will check that exp1 has been garbage collected correctly self.assertIsNone(train_dataset_exp_0_weak_ref()) self.assertIsNone(train_dataset_exp_1_weak_ref()) with self.assertRaises(Exception): exp_0 = benchmark_instance.train_stream[0] with self.assertRaises(Exception): exp_1 = benchmark_instance.train_stream[1]
def test_lazy_scenario(self): train_exps, test_exps, other_stream_exps = self._make_tensor_datasets() def train_gen(): # Lazy generator of the training stream for dataset in train_exps: yield dataset def test_gen(): # Lazy generator of the test stream for dataset in test_exps: yield dataset def other_gen(): # Lazy generator of the "other" stream for dataset in other_stream_exps: yield dataset benchmark_instance = GenericCLScenario(stream_definitions=dict( train=((train_gen(), len(train_exps)), [ train_exps[0].targets_task_labels, train_exps[1].targets_task_labels ]), test=((test_gen(), len(test_exps)), [test_exps[0].targets_task_labels]), other=((other_gen(), len(other_stream_exps)), [other_stream_exps[0].targets_task_labels]))) # --- START: Test classes timeline before first experience --- current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertIsNone(current_classes) self.assertSetEqual(set(), set(prev_classes)) self.assertIsNone(cumulative_classes) self.assertIsNone(future_classes) # --- END: Test classes timeline before first experience --- train_exp_0: GenericExperience = benchmark_instance.train_stream[0] # --- START: Test classes timeline at first experience --- current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertSetEqual(set(train_exps[0].targets), set(current_classes)) self.assertSetEqual(set(), set(prev_classes)) self.assertSetEqual(set(train_exps[0].targets), set(cumulative_classes)) self.assertIsNone(future_classes) current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(1) self.assertIsNone(current_classes) self.assertSetEqual(set(train_exps[0].targets), set(prev_classes)) # None because we didn't load exp 0 yet self.assertIsNone(cumulative_classes) self.assertSetEqual(set(), set(future_classes)) # --- END: Test classes timeline at first experience --- train_exp_1: GenericExperience = benchmark_instance.train_stream[1] # --- START: Test classes timeline at second experience --- # Check if get_classes_timeline(0) is consistent current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(0) self.assertSetEqual(set(train_exps[0].targets), set(current_classes)) self.assertSetEqual(set(), set(prev_classes)) self.assertSetEqual(set(train_exps[0].targets), set(cumulative_classes)) # We now have access to future classes! self.assertSetEqual(set(train_exps[1].targets), set(future_classes)) current_classes, prev_classes, cumulative_classes, future_classes = \ benchmark_instance.get_classes_timeline(1) self.assertSetEqual(set(train_exps[1].targets), set(current_classes)) self.assertSetEqual(set(train_exps[0].targets), set(prev_classes)) self.assertSetEqual( set(train_exps[0].targets).union(set(train_exps[1].targets)), set(cumulative_classes)) self.assertSetEqual(set(), set(future_classes)) # --- END: Test classes timeline at second experience --- train_0_classes = train_exp_0.classes_in_this_experience train_1_classes = train_exp_1.classes_in_this_experience train_0_classes_min = min(train_0_classes) train_1_classes_min = min(train_1_classes) train_0_classes_max = max(train_0_classes) train_1_classes_max = max(train_1_classes) self.assertGreaterEqual(train_0_classes_min, 0) self.assertLess(train_0_classes_max, 70) self.assertGreaterEqual(train_1_classes_min, 0) self.assertLess(train_1_classes_max, 100) with self.assertRaises(IndexError): train_exp_2: GenericExperience = benchmark_instance.train_stream[2] test_exp_0: GenericExperience = benchmark_instance.test_stream[0] test_0_classes = test_exp_0.classes_in_this_experience test_0_classes_min = min(test_0_classes) test_0_classes_max = max(test_0_classes) self.assertGreaterEqual(test_0_classes_min, 100) self.assertLess(test_0_classes_max, 200) with self.assertRaises(IndexError): test_exp_1: GenericExperience = benchmark_instance.test_stream[1] other_exp_0: GenericExperience = benchmark_instance.other_stream[0] other_0_classes = other_exp_0.classes_in_this_experience other_0_classes_min = min(other_0_classes) other_0_classes_max = max(other_0_classes) self.assertGreaterEqual(other_0_classes_min, 400) self.assertLess(other_0_classes_max, 600) with self.assertRaises(IndexError): other_exp_1: GenericExperience = benchmark_instance.other_stream[1]