def test_complete_test_set_only(self):
train_exps = []
test_exps = []
for _ in range(5):
tensor_x = torch.rand(200, 3, 28, 28)
tensor_y = torch.randint(0, 100, (200, ))
tensor_t = torch.randint(0, 5, (200, ))
train_exps.append(
AvalancheTensorDataset(tensor_x,
tensor_y,
task_labels=tensor_t))
for _ in range(3):
tensor_x = torch.rand(150, 3, 28, 28)
tensor_y = torch.randint(0, 100, (150, ))
tensor_t = torch.randint(0, 5, (150, ))
test_exps.append(
AvalancheTensorDataset(tensor_x,
tensor_y,
task_labels=tensor_t))
with self.assertRaises(Exception):
benchmark_instance = GenericCLScenario(
stream_definitions={
"train": (train_exps, ),
"test": (test_exps, ),
},
complete_test_set_only=True,
)
benchmark_instance = GenericCLScenario(
stream_definitions={
"train": (train_exps, ),
"test": (test_exps[0], ),
},
complete_test_set_only=True,
)
self.assertEqual(5, len(benchmark_instance.train_stream))
self.assertEqual(1, len(benchmark_instance.test_stream))
def test_custom_streams_name_and_length(self):
train_exps = []
test_exps = []
valid_exps = []
for _ in range(5):
tensor_x = torch.rand(200, 3, 28, 28)
tensor_y = torch.randint(0, 100, (200,))
tensor_t = torch.randint(0, 5, (200,))
train_exps.append(
AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t)
)
for _ in range(3):
tensor_x = torch.rand(150, 3, 28, 28)
tensor_y = torch.randint(0, 100, (150,))
tensor_t = torch.randint(0, 3, (150,))
test_exps.append(
AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t)
)
for _ in range(4):
tensor_x = torch.rand(220, 3, 28, 28)
tensor_y = torch.randint(0, 100, (220,))
tensor_t = torch.randint(0, 5, (220,))
valid_exps.append(
AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t)
)
valid_origin_dataset = AvalancheTensorDataset(
torch.ones(10, 3, 32, 32), torch.zeros(10)
)
valid_t_labels = [{9}, {4, 5}, {7, 8}, {0}, {3}]
with self.assertRaises(Exception):
benchmark_instance = GenericCLScenario(
stream_definitions={
"train": (train_exps,),
"test": (test_exps,),
"valid": (valid_exps, valid_t_labels, valid_origin_dataset),
}
)
valid_t_labels = valid_t_labels[:-1]
benchmark_instance = GenericCLScenario(
stream_definitions={
"train": (train_exps,),
"test": (test_exps,),
"valid": (valid_exps, valid_t_labels, valid_origin_dataset),
}
)
self.assertEqual(5, len(benchmark_instance.train_stream))
self.assertEqual(3, len(benchmark_instance.test_stream))
self.assertEqual(4, len(benchmark_instance.valid_stream))
self.assertEqual(None, benchmark_instance.original_train_dataset)
self.assertEqual(None, benchmark_instance.original_test_dataset)
self.assertEqual(
valid_origin_dataset, benchmark_instance.original_valid_dataset
)
for i, exp in enumerate(benchmark_instance.train_stream):
expect_x, expect_y, expect_t = train_exps[i][0]
got_x, got_y, got_t = exp.dataset[0]
self.assertTrue(torch.equal(expect_x, got_x))
self.assertTrue(torch.equal(expect_y, got_y))
self.assertEqual(int(expect_t), got_t)
exp_t_labels = set(exp.task_labels)
self.assertLess(max(exp_t_labels), 5)
self.assertGreaterEqual(min(exp_t_labels), 0)
for i, exp in enumerate(benchmark_instance.test_stream):
expect_x, expect_y, expect_t = test_exps[i][0]
got_x, got_y, got_t = exp.dataset[0]
self.assertTrue(torch.equal(expect_x, got_x))
self.assertTrue(torch.equal(expect_y, got_y))
self.assertEqual(int(expect_t), got_t)
exp_t_labels = set(exp.task_labels)
self.assertLess(max(exp_t_labels), 3)
self.assertGreaterEqual(min(exp_t_labels), 0)
for i, exp in enumerate(benchmark_instance.valid_stream):
expect_x, expect_y, expect_t = valid_exps[i][0]
got_x, got_y, got_t = exp.dataset[0]
self.assertTrue(torch.equal(expect_x, got_x))
self.assertTrue(torch.equal(expect_y, got_y))
self.assertEqual(int(expect_t), got_t)
exp_t_labels = set(exp.task_labels)
self.assertEqual(valid_t_labels[i], exp_t_labels)
def test_lazy_scenario_drop_old_ones(self):
train_exps, test_exps, other_stream_exps = self._make_tensor_datasets()
train_dataset_exp_0_weak_ref = weakref.ref(train_exps[0])
train_dataset_exp_1_weak_ref = weakref.ref(train_exps[1])
train_gen = GenericCLScenarioTests._generate_stream(train_exps)
test_gen = GenericCLScenarioTests._generate_stream(test_exps)
other_gen = GenericCLScenarioTests._generate_stream(other_stream_exps)
benchmark_instance = GenericCLScenario(stream_definitions=dict(
train=((train_gen, len(train_exps)), [
train_exps[0].targets_task_labels,
train_exps[1].targets_task_labels
]),
test=((test_gen, len(test_exps)),
[test_exps[0].targets_task_labels]),
other=((other_gen, len(other_stream_exps)),
[other_stream_exps[0].targets_task_labels])))
# --- START: Test classes timeline before first experience ---
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertIsNone(current_classes)
self.assertSetEqual(set(), set(prev_classes))
self.assertIsNone(cumulative_classes)
self.assertIsNone(future_classes)
# --- END: Test classes timeline before first experience ---
train_exp_0: GenericExperience = benchmark_instance.train_stream[0]
# --- START: Test classes timeline at first experience ---
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertSetEqual(set(train_exps[0].targets), set(current_classes))
self.assertSetEqual(set(), set(prev_classes))
self.assertSetEqual(set(train_exps[0].targets),
set(cumulative_classes))
self.assertIsNone(future_classes)
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(1)
self.assertIsNone(current_classes)
self.assertSetEqual(set(train_exps[0].targets), set(prev_classes))
# None because we didn't load exp 0 yet
self.assertIsNone(cumulative_classes)
self.assertSetEqual(set(), set(future_classes))
# --- END: Test classes timeline at first experience ---
# Check if it works when the previous experience is dropped
benchmark_instance.train_stream.drop_previous_experiences(0)
train_exp_1: GenericExperience = benchmark_instance.train_stream[1]
# --- START: Test classes timeline at second experience ---
# Check if get_classes_timeline(0) is consistent
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertSetEqual(set(train_exps[0].targets), set(current_classes))
self.assertSetEqual(set(), set(prev_classes))
self.assertSetEqual(set(train_exps[0].targets),
set(cumulative_classes))
# We now have access to future classes!
self.assertSetEqual(set(train_exps[1].targets), set(future_classes))
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(1)
self.assertSetEqual(set(train_exps[1].targets), set(current_classes))
self.assertSetEqual(set(train_exps[0].targets), set(prev_classes))
self.assertSetEqual(
set(train_exps[0].targets).union(set(train_exps[1].targets)),
set(cumulative_classes))
self.assertSetEqual(set(), set(future_classes))
# --- END: Test classes timeline at second experience ---
train_0_classes = train_exp_0.classes_in_this_experience
train_1_classes = train_exp_1.classes_in_this_experience
train_0_classes_min = min(train_0_classes)
train_1_classes_min = min(train_1_classes)
train_0_classes_max = max(train_0_classes)
train_1_classes_max = max(train_1_classes)
self.assertGreaterEqual(train_0_classes_min, 0)
self.assertLess(train_0_classes_max, 70)
self.assertGreaterEqual(train_1_classes_min, 0)
self.assertLess(train_1_classes_max, 100)
with self.assertRaises(IndexError):
train_exp_2: GenericExperience = benchmark_instance.train_stream[2]
test_exp_0: GenericExperience = benchmark_instance.test_stream[0]
test_0_classes = test_exp_0.classes_in_this_experience
test_0_classes_min = min(test_0_classes)
test_0_classes_max = max(test_0_classes)
self.assertGreaterEqual(test_0_classes_min, 100)
self.assertLess(test_0_classes_max, 200)
with self.assertRaises(IndexError):
test_exp_1: GenericExperience = benchmark_instance.test_stream[1]
other_exp_0: GenericExperience = benchmark_instance.other_stream[0]
other_0_classes = other_exp_0.classes_in_this_experience
other_0_classes_min = min(other_0_classes)
other_0_classes_max = max(other_0_classes)
self.assertGreaterEqual(other_0_classes_min, 400)
self.assertLess(other_0_classes_max, 600)
with self.assertRaises(IndexError):
other_exp_1: GenericExperience = benchmark_instance.other_stream[1]
train_exps = None
train_exp_0 = None
train_exp_1 = None
train_0_classes = None
train_1_classes = None
train_gen = None
# The generational GC is needed, ref-count is not enough here
gc.collect()
# This will check that the train dataset of exp0 has been garbage
# collected correctly
self.assertIsNone(train_dataset_exp_0_weak_ref())
self.assertIsNotNone(train_dataset_exp_1_weak_ref())
benchmark_instance.train_stream.drop_previous_experiences(1)
gc.collect()
# This will check that exp1 has been garbage collected correctly
self.assertIsNone(train_dataset_exp_0_weak_ref())
self.assertIsNone(train_dataset_exp_1_weak_ref())
with self.assertRaises(Exception):
exp_0 = benchmark_instance.train_stream[0]
with self.assertRaises(Exception):
exp_1 = benchmark_instance.train_stream[1]
def test_lazy_scenario(self):
train_exps, test_exps, other_stream_exps = self._make_tensor_datasets()
def train_gen():
# Lazy generator of the training stream
for dataset in train_exps:
yield dataset
def test_gen():
# Lazy generator of the test stream
for dataset in test_exps:
yield dataset
def other_gen():
# Lazy generator of the "other" stream
for dataset in other_stream_exps:
yield dataset
benchmark_instance = GenericCLScenario(stream_definitions=dict(
train=((train_gen(), len(train_exps)), [
train_exps[0].targets_task_labels,
train_exps[1].targets_task_labels
]),
test=((test_gen(), len(test_exps)),
[test_exps[0].targets_task_labels]),
other=((other_gen(), len(other_stream_exps)),
[other_stream_exps[0].targets_task_labels])))
# --- START: Test classes timeline before first experience ---
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertIsNone(current_classes)
self.assertSetEqual(set(), set(prev_classes))
self.assertIsNone(cumulative_classes)
self.assertIsNone(future_classes)
# --- END: Test classes timeline before first experience ---
train_exp_0: GenericExperience = benchmark_instance.train_stream[0]
# --- START: Test classes timeline at first experience ---
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertSetEqual(set(train_exps[0].targets), set(current_classes))
self.assertSetEqual(set(), set(prev_classes))
self.assertSetEqual(set(train_exps[0].targets),
set(cumulative_classes))
self.assertIsNone(future_classes)
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(1)
self.assertIsNone(current_classes)
self.assertSetEqual(set(train_exps[0].targets), set(prev_classes))
# None because we didn't load exp 0 yet
self.assertIsNone(cumulative_classes)
self.assertSetEqual(set(), set(future_classes))
# --- END: Test classes timeline at first experience ---
train_exp_1: GenericExperience = benchmark_instance.train_stream[1]
# --- START: Test classes timeline at second experience ---
# Check if get_classes_timeline(0) is consistent
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(0)
self.assertSetEqual(set(train_exps[0].targets), set(current_classes))
self.assertSetEqual(set(), set(prev_classes))
self.assertSetEqual(set(train_exps[0].targets),
set(cumulative_classes))
# We now have access to future classes!
self.assertSetEqual(set(train_exps[1].targets), set(future_classes))
current_classes, prev_classes, cumulative_classes, future_classes = \
benchmark_instance.get_classes_timeline(1)
self.assertSetEqual(set(train_exps[1].targets), set(current_classes))
self.assertSetEqual(set(train_exps[0].targets), set(prev_classes))
self.assertSetEqual(
set(train_exps[0].targets).union(set(train_exps[1].targets)),
set(cumulative_classes))
self.assertSetEqual(set(), set(future_classes))
# --- END: Test classes timeline at second experience ---
train_0_classes = train_exp_0.classes_in_this_experience
train_1_classes = train_exp_1.classes_in_this_experience
train_0_classes_min = min(train_0_classes)
train_1_classes_min = min(train_1_classes)
train_0_classes_max = max(train_0_classes)
train_1_classes_max = max(train_1_classes)
self.assertGreaterEqual(train_0_classes_min, 0)
self.assertLess(train_0_classes_max, 70)
self.assertGreaterEqual(train_1_classes_min, 0)
self.assertLess(train_1_classes_max, 100)
with self.assertRaises(IndexError):
train_exp_2: GenericExperience = benchmark_instance.train_stream[2]
test_exp_0: GenericExperience = benchmark_instance.test_stream[0]
test_0_classes = test_exp_0.classes_in_this_experience
test_0_classes_min = min(test_0_classes)
test_0_classes_max = max(test_0_classes)
self.assertGreaterEqual(test_0_classes_min, 100)
self.assertLess(test_0_classes_max, 200)
with self.assertRaises(IndexError):
test_exp_1: GenericExperience = benchmark_instance.test_stream[1]
other_exp_0: GenericExperience = benchmark_instance.other_stream[0]
other_0_classes = other_exp_0.classes_in_this_experience
other_0_classes_min = min(other_0_classes)
other_0_classes_max = max(other_0_classes)
self.assertGreaterEqual(other_0_classes_min, 400)
self.assertLess(other_0_classes_max, 600)
with self.assertRaises(IndexError):
other_exp_1: GenericExperience = benchmark_instance.other_stream[1]
def benchmark_with_validation_stream(
benchmark_instance: GenericCLScenario,
validation_size: Union[int, float],
shuffle: bool = False,
input_stream: str = 'train',
output_stream: str = 'valid',
custom_split_strategy: Callable[[Experience],
Tuple[AvalancheDataset,
AvalancheDataset]] = None,
*,
experience_factory: Callable[[GenericScenarioStream, int],
Experience] = None,
lazy_splitting: bool = None):
"""
Helper that can be used to obtain a benchmark with a validation stream.
This generator accepts an existing benchmark instance and returns a version
of it in which a validation stream has been added.
In its base form this generator will split train experiences to extract
validation experiences of a fixed (by number of instances or relative
size), configurable, size. The split can be also performed on other
streams if needed and the name of the resulting validation stream can
be configured too.
Each validation experience will be extracted directly from a single training
experience. Patterns selected for the validation experience will be removed
from the training one.
If shuffle is True, the validation stream will be created randomly.
Beware that no kind of class balancing is done.
The `custom_split_strategy` parameter can be used if a more specific
splitting is required.
Please note that the resulting experiences will have a task labels field
equal to the one of the originating experience.
Experience splitting can be executed in a lazy way. This behavior can be
controlled using the `lazy_splitting` parameter. By default, experiences
are split in a lazy way only when the input stream is lazily generated.
:param benchmark_instance: The benchmark to split.
:param validation_size: The size of the validation experience, as an int
or a float between 0 and 1. Ignored if `custom_split_strategy` is used.
:param shuffle: If True, patterns will be allocated to the validation
stream randomly. This will use the default PyTorch random number
generator at its current state. Defaults to False. Ignored if
`custom_split_strategy` is used. If False, the first instances will be
allocated to the training dataset by leaving the last ones to the
validation dataset.
:param input_stream: The name of the input stream. Defaults to 'train'.
:param output_stream: The name of the output stream. Defaults to 'valid'.
:param custom_split_strategy: A function that implements a custom splitting
strategy. The function must accept an experience and return a tuple
containing the new train and validation dataset. Defaults to None,
which means that the standard splitting strategy will be used (which
creates experiences according to `validation_size` and `shuffle`).
A good starting to understand the mechanism is to look at the
implementation of the standard splitting function
:func:`random_validation_split_strategy`.
:param experience_factory: The experience factory. Defaults to
:class:`GenericExperience`.
:param lazy_splitting: If True, the stream will be split in a lazy way.
If False, the stream will be split immediately. Defaults to None, which
means that the stream will be split in a lazy or non-lazy way depending
on the laziness of the `input_stream`.
:return: A benchmark instance in which the validation stream has been added.
"""
split_strategy = custom_split_strategy
if split_strategy is None:
split_strategy = partial(
random_validation_split_strategy, validation_size,
shuffle)
stream_definitions: TStreamsUserDict = dict(
benchmark_instance.stream_definitions)
streams = benchmark_instance.streams
if input_stream not in streams:
raise ValueError(f'Stream {input_stream} could not be found in the '
f'benchmark instance')
if output_stream in streams:
raise ValueError(f'Stream {output_stream} already exists in the '
f'benchmark instance')
stream = streams[input_stream]
split_lazily = lazy_splitting
if split_lazily is None:
split_lazily = stream_definitions[input_stream].is_lazy
exps_tasks_labels = list(
stream_definitions[input_stream].exps_task_labels
)
if not split_lazily:
# Classic static splitting
train_exps_source = []
valid_exps_source = []
exp: Experience
for exp in stream:
train_exp, valid_exp = split_strategy(exp)
train_exps_source.append(train_exp)
valid_exps_source.append(valid_exp)
else:
# Lazy splitting (based on a generator)
split_generator = _lazy_train_val_split(split_strategy, stream)
train_exps_gen, valid_exps_gen = _gen_split(split_generator)
train_exps_source = (train_exps_gen, len(stream))
valid_exps_source = (valid_exps_gen, len(stream))
train_stream_def = \
StreamUserDef(
train_exps_source,
exps_tasks_labels,
stream_definitions[input_stream].origin_dataset,
split_lazily)
valid_stream_def = \
StreamUserDef(
valid_exps_source,
exps_tasks_labels,
stream_definitions[input_stream].origin_dataset,
split_lazily)
stream_definitions[input_stream] = train_stream_def
stream_definitions[output_stream] = valid_stream_def
complete_test_set_only = benchmark_instance.complete_test_set_only
return GenericCLScenario(stream_definitions=stream_definitions,
complete_test_set_only=complete_test_set_only,
experience_factory=experience_factory)
def data_incremental_benchmark(
benchmark_instance: GenericCLScenario,
experience_size: int,
shuffle: bool = False,
drop_last: bool = False,
split_streams: Sequence[str] = ('train',),
custom_split_strategy: Callable[[Experience],
Sequence[AvalancheDataset]] = None,
experience_factory: Callable[[GenericScenarioStream, int],
Experience] = None):
"""
High-level benchmark generator for a Data Incremental setup.
This generator accepts an existing benchmark instance and returns a version
of it in which experiences have been split in order to produce a
Data Incremental stream.
In its base form this generator will split train experiences in experiences
of a fixed, configurable, size. The split can be also performed on other
streams (like the test one) if needed.
The `custom_split_strategy` parameter can be used if a more specific
splitting is required.
Beware that experience splitting is NOT executed in a lazy way. This
means that the splitting process takes place immediately. Consider
optimizing the split process for speed when using a custom splitting
strategy.
Please note that each mini-experience will have a task labels field
equal to the one of the originating experience.
The `complete_test_set_only` field of the resulting benchmark instance
will be `True` only if the same field of original benchmark instance is
`True` and if the resulting test stream contains exactly one experience.
:param benchmark_instance: The benchmark to split.
:param experience_size: The size of the experience, as an int. Ignored
if `custom_split_strategy` is used.
:param shuffle: If True, experiences will be split by first shuffling
instances in each experience. This will use the default PyTorch
random number generator at its current state. Defaults to False.
Ignored if `custom_split_strategy` is used.
:param drop_last: If True, if the last experience doesn't contain
`experience_size` instances, then the last experience will be dropped.
Defaults to False. Ignored if `custom_split_strategy` is used.
:param split_streams: The list of streams to split. By default only the
"train" stream will be split.
:param custom_split_strategy: A function that implements a custom splitting
strategy. The function must accept an experience and return a list
of datasets each describing an experience. Defaults to None, which means
that the standard splitting strategy will be used (which creates
experiences of size `experience_size`).
A good starting to understand the mechanism is to look at the
implementation of the standard splitting function
:func:`fixed_size_experience_split_strategy`.
:param experience_factory: The experience factory.
Defaults to :class:`GenericExperience`.
:return: The Data Incremental benchmark instance.
"""
split_strategy = custom_split_strategy
if split_strategy is None:
split_strategy = partial(
fixed_size_experience_split_strategy, experience_size, shuffle,
drop_last)
stream_definitions: TStreamsUserDict = dict(
benchmark_instance.stream_definitions)
for stream_name in split_streams:
if stream_name not in stream_definitions:
raise ValueError(f'Stream {stream_name} could not be found in the '
f'benchmark instance')
stream = getattr(benchmark_instance, f'{stream_name}_stream')
split_datasets: List[AvalancheDataset] = []
split_task_labels: List[Set[int]] = []
exp: Experience
for exp in stream:
experiences = split_strategy(exp)
split_datasets += experiences
for _ in range(len(experiences)):
split_task_labels.append(set(exp.task_labels))
stream_def = StreamUserDef(
split_datasets, split_task_labels,
stream_definitions[stream_name].origin_dataset,
False)
stream_definitions[stream_name] = stream_def
complete_test_set_only = benchmark_instance.complete_test_set_only and \
len(stream_definitions['test'].exps_data) == 1
return GenericCLScenario(stream_definitions=stream_definitions,
complete_test_set_only=complete_test_set_only,
experience_factory=experience_factory)
