def test_lazy_scenario_drop_old_ones(self):
train_exps, test_exps, other_stream_exps = self._make_tensor_datasets()
train_dataset_exp_0_weak_ref = weakref.ref(train_exps[0])
train_dataset_exp_1_weak_ref = weakref.ref(train_exps[1])
train_gen = GenericCLScenarioTests._generate_stream(train_exps)
test_gen = GenericCLScenarioTests._generate_stream(test_exps)
other_gen = GenericCLScenarioTests._generate_stream(other_stream_exps)
benchmark_instance = GenericCLScenario(stream_definitions=dict(
train=((train_gen, len(train_exps)), [
train_exps[0].targets_task_labels,
train_exps[1].targets_task_labels
]),
test=((test_gen, len(test_exps)),
[test_exps[0].targets_task_labels]),
other=((other_gen, len(other_stream_exps)),
[other_stream_exps[0].targets_task_labels])))
# --- START: Test classes timeline before first experience ---
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertIsNone(current_classes)
self.assertSetEqual(set(), set(prev_classes))
self.assertIsNone(cumulative_classes)
self.assertIsNone(future_classes)
# --- END: Test classes timeline before first experience ---
train_exp_0: GenericExperience = benchmark_instance.train_stream[0]
# --- START: Test classes timeline at first experience ---
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertSetEqual(set(train_exps[0].targets), set(current_classes))
self.assertSetEqual(set(), set(prev_classes))
self.assertSetEqual(set(train_exps[0].targets),
set(cumulative_classes))
self.assertIsNone(future_classes)
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(1)
self.assertIsNone(current_classes)
self.assertSetEqual(set(train_exps[0].targets), set(prev_classes))
# None because we didn't load exp 0 yet
self.assertIsNone(cumulative_classes)
self.assertSetEqual(set(), set(future_classes))
# --- END: Test classes timeline at first experience ---
# Check if it works when the previous experience is dropped
benchmark_instance.train_stream.drop_previous_experiences(0)
train_exp_1: GenericExperience = benchmark_instance.train_stream[1]
# --- START: Test classes timeline at second experience ---
# Check if get_classes_timeline(0) is consistent
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertSetEqual(set(train_exps[0].targets), set(current_classes))
self.assertSetEqual(set(), set(prev_classes))
self.assertSetEqual(set(train_exps[0].targets),
set(cumulative_classes))
# We now have access to future classes!
self.assertSetEqual(set(train_exps[1].targets), set(future_classes))
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(1)
self.assertSetEqual(set(train_exps[1].targets), set(current_classes))
self.assertSetEqual(set(train_exps[0].targets), set(prev_classes))
self.assertSetEqual(
set(train_exps[0].targets).union(set(train_exps[1].targets)),
set(cumulative_classes))
self.assertSetEqual(set(), set(future_classes))
# --- END: Test classes timeline at second experience ---
train_0_classes = train_exp_0.classes_in_this_experience
train_1_classes = train_exp_1.classes_in_this_experience
train_0_classes_min = min(train_0_classes)
train_1_classes_min = min(train_1_classes)
train_0_classes_max = max(train_0_classes)
train_1_classes_max = max(train_1_classes)
self.assertGreaterEqual(train_0_classes_min, 0)
self.assertLess(train_0_classes_max, 70)
self.assertGreaterEqual(train_1_classes_min, 0)
self.assertLess(train_1_classes_max, 100)
with self.assertRaises(IndexError):
train_exp_2: GenericExperience = benchmark_instance.train_stream[2]
test_exp_0: GenericExperience = benchmark_instance.test_stream[0]
test_0_classes = test_exp_0.classes_in_this_experience
test_0_classes_min = min(test_0_classes)
test_0_classes_max = max(test_0_classes)
self.assertGreaterEqual(test_0_classes_min, 100)
self.assertLess(test_0_classes_max, 200)
with self.assertRaises(IndexError):
test_exp_1: GenericExperience = benchmark_instance.test_stream[1]
other_exp_0: GenericExperience = benchmark_instance.other_stream[0]
other_0_classes = other_exp_0.classes_in_this_experience
other_0_classes_min = min(other_0_classes)
other_0_classes_max = max(other_0_classes)
self.assertGreaterEqual(other_0_classes_min, 400)
self.assertLess(other_0_classes_max, 600)
with self.assertRaises(IndexError):
other_exp_1: GenericExperience = benchmark_instance.other_stream[1]
train_exps = None
train_exp_0 = None
train_exp_1 = None
train_0_classes = None
train_1_classes = None
train_gen = None
# The generational GC is needed, ref-count is not enough here
gc.collect()
# This will check that the train dataset of exp0 has been garbage
# collected correctly
self.assertIsNone(train_dataset_exp_0_weak_ref())
self.assertIsNotNone(train_dataset_exp_1_weak_ref())
benchmark_instance.train_stream.drop_previous_experiences(1)
gc.collect()
# This will check that exp1 has been garbage collected correctly
self.assertIsNone(train_dataset_exp_0_weak_ref())
self.assertIsNone(train_dataset_exp_1_weak_ref())
with self.assertRaises(Exception):
exp_0 = benchmark_instance.train_stream[0]
with self.assertRaises(Exception):
exp_1 = benchmark_instance.train_stream[1]
def test_lazy_scenario(self):
train_exps, test_exps, other_stream_exps = self._make_tensor_datasets()
def train_gen():
# Lazy generator of the training stream
for dataset in train_exps:
yield dataset
def test_gen():
# Lazy generator of the test stream
for dataset in test_exps:
yield dataset
def other_gen():
# Lazy generator of the "other" stream
for dataset in other_stream_exps:
yield dataset
benchmark_instance = GenericCLScenario(stream_definitions=dict(
train=((train_gen(), len(train_exps)), [
train_exps[0].targets_task_labels,
train_exps[1].targets_task_labels
]),
test=((test_gen(), len(test_exps)),
[test_exps[0].targets_task_labels]),
other=((other_gen(), len(other_stream_exps)),
[other_stream_exps[0].targets_task_labels])))
# --- START: Test classes timeline before first experience ---
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertIsNone(current_classes)
self.assertSetEqual(set(), set(prev_classes))
self.assertIsNone(cumulative_classes)
self.assertIsNone(future_classes)
# --- END: Test classes timeline before first experience ---
train_exp_0: GenericExperience = benchmark_instance.train_stream[0]
# --- START: Test classes timeline at first experience ---
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertSetEqual(set(train_exps[0].targets), set(current_classes))
self.assertSetEqual(set(), set(prev_classes))
self.assertSetEqual(set(train_exps[0].targets),
set(cumulative_classes))
self.assertIsNone(future_classes)
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(1)
self.assertIsNone(current_classes)
self.assertSetEqual(set(train_exps[0].targets), set(prev_classes))
# None because we didn't load exp 0 yet
self.assertIsNone(cumulative_classes)
self.assertSetEqual(set(), set(future_classes))
# --- END: Test classes timeline at first experience ---
train_exp_1: GenericExperience = benchmark_instance.train_stream[1]
# --- START: Test classes timeline at second experience ---
# Check if get_classes_timeline(0) is consistent
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertSetEqual(set(train_exps[0].targets), set(current_classes))
self.assertSetEqual(set(), set(prev_classes))
self.assertSetEqual(set(train_exps[0].targets),
set(cumulative_classes))
# We now have access to future classes!
self.assertSetEqual(set(train_exps[1].targets), set(future_classes))
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(1)
self.assertSetEqual(set(train_exps[1].targets), set(current_classes))
self.assertSetEqual(set(train_exps[0].targets), set(prev_classes))
self.assertSetEqual(
set(train_exps[0].targets).union(set(train_exps[1].targets)),
set(cumulative_classes))
self.assertSetEqual(set(), set(future_classes))
# --- END: Test classes timeline at second experience ---
train_0_classes = train_exp_0.classes_in_this_experience
train_1_classes = train_exp_1.classes_in_this_experience
train_0_classes_min = min(train_0_classes)
train_1_classes_min = min(train_1_classes)
train_0_classes_max = max(train_0_classes)
train_1_classes_max = max(train_1_classes)
self.assertGreaterEqual(train_0_classes_min, 0)
self.assertLess(train_0_classes_max, 70)
self.assertGreaterEqual(train_1_classes_min, 0)
self.assertLess(train_1_classes_max, 100)
with self.assertRaises(IndexError):
train_exp_2: GenericExperience = benchmark_instance.train_stream[2]
test_exp_0: GenericExperience = benchmark_instance.test_stream[0]
test_0_classes = test_exp_0.classes_in_this_experience
test_0_classes_min = min(test_0_classes)
test_0_classes_max = max(test_0_classes)
self.assertGreaterEqual(test_0_classes_min, 100)
self.assertLess(test_0_classes_max, 200)
with self.assertRaises(IndexError):
test_exp_1: GenericExperience = benchmark_instance.test_stream[1]
other_exp_0: GenericExperience = benchmark_instance.other_stream[0]
other_0_classes = other_exp_0.classes_in_this_experience
other_0_classes_min = min(other_0_classes)
other_0_classes_max = max(other_0_classes)
self.assertGreaterEqual(other_0_classes_min, 400)
self.assertLess(other_0_classes_max, 600)
with self.assertRaises(IndexError):
other_exp_1: GenericExperience = benchmark_instance.other_stream[1]