def _prepare_rng_critical_parts(seed=1234):
torch.random.manual_seed(seed)
initial_benchmark = PluginTests.create_benchmark(seed=seed)
val_benchmark = benchmark_with_validation_stream(
initial_benchmark, 0.3, shuffle=True
)
return (val_benchmark, _PlainMLP(input_size=6, hidden_size=10))
def test_lazy_benchmark_with_validation_stream_fixed_size(self):
lazy_options = [None, True, False]
for lazy_option in lazy_options:
with self.subTest(lazy_option=lazy_option):
pattern_shape = (3, 32, 32)
# Definition of training experiences
# Experience 1
experience_1_x = torch.zeros(100, *pattern_shape)
experience_1_y = torch.zeros(100, dtype=torch.long)
experience_1_dataset = AvalancheTensorDataset(
experience_1_x, experience_1_y)
# Experience 2
experience_2_x = torch.zeros(80, *pattern_shape)
experience_2_y = torch.ones(80, dtype=torch.long)
experience_2_dataset = AvalancheTensorDataset(
experience_2_x, experience_2_y)
# Test experience
test_x = torch.zeros(50, *pattern_shape)
test_y = torch.zeros(50, dtype=torch.long)
experience_test = AvalancheTensorDataset(test_x, test_y)
def train_gen():
# Lazy generator of the training stream
for dataset in [
experience_1_dataset, experience_2_dataset
]:
yield dataset
def test_gen():
# Lazy generator of the test stream
for dataset in [experience_test]:
yield dataset
initial_benchmark_instance = create_lazy_generic_benchmark(
train_generator=LazyStreamDefinition(
train_gen(), 2, [0, 0]),
test_generator=LazyStreamDefinition(test_gen(), 1, [0]),
complete_test_set_only=True,
dataset_type=AvalancheDatasetType.CLASSIFICATION,
)
valid_benchmark = benchmark_with_validation_stream(
initial_benchmark_instance,
20,
shuffle=False,
lazy_splitting=lazy_option,
)
if lazy_option is None or lazy_option:
expect_laziness = True
else:
expect_laziness = False
self.assertEqual(
expect_laziness,
valid_benchmark.stream_definitions["train"].is_lazy,
)
self.assertEqual(2, len(valid_benchmark.train_stream))
self.assertEqual(2, len(valid_benchmark.valid_stream))
self.assertEqual(1, len(valid_benchmark.test_stream))
self.assertTrue(valid_benchmark.complete_test_set_only)
maybe_exp = valid_benchmark.stream_definitions[
"train"].exps_data.get_experience_if_loaded(0)
self.assertEqual(expect_laziness, maybe_exp is None)
self.assertEqual(80,
len(valid_benchmark.train_stream[0].dataset))
maybe_exp = valid_benchmark.stream_definitions[
"train"].exps_data.get_experience_if_loaded(1)
self.assertEqual(expect_laziness, maybe_exp is None)
self.assertEqual(60,
len(valid_benchmark.train_stream[1].dataset))
maybe_exp = valid_benchmark.stream_definitions[
"valid"].exps_data.get_experience_if_loaded(0)
self.assertEqual(expect_laziness, maybe_exp is None)
self.assertEqual(20,
len(valid_benchmark.valid_stream[0].dataset))
maybe_exp = valid_benchmark.stream_definitions[
"valid"].exps_data.get_experience_if_loaded(1)
self.assertEqual(expect_laziness, maybe_exp is None)
self.assertEqual(20,
len(valid_benchmark.valid_stream[1].dataset))
self.assertIsNotNone(
valid_benchmark.stream_definitions["train"].exps_data.
get_experience_if_loaded(0))
self.assertIsNotNone(
valid_benchmark.stream_definitions["valid"].exps_data.
get_experience_if_loaded(0))
self.assertIsNotNone(
valid_benchmark.stream_definitions["train"].exps_data.
get_experience_if_loaded(1))
self.assertIsNotNone(
valid_benchmark.stream_definitions["valid"].exps_data.
get_experience_if_loaded(1))
self.assertTrue(
torch.equal(
experience_1_x[:80],
valid_benchmark.train_stream[0].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_2_x[:60],
valid_benchmark.train_stream[1].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_1_y[:80],
valid_benchmark.train_stream[0].dataset[:][1],
))
self.assertTrue(
torch.equal(
experience_2_y[:60],
valid_benchmark.train_stream[1].dataset[:][1],
))
self.assertTrue(
torch.equal(
experience_1_x[80:],
valid_benchmark.valid_stream[0].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_2_x[60:],
valid_benchmark.valid_stream[1].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_1_y[80:],
valid_benchmark.valid_stream[0].dataset[:][1],
))
self.assertTrue(
torch.equal(
experience_2_y[60:],
valid_benchmark.valid_stream[1].dataset[:][1],
))
self.assertTrue(
torch.equal(test_x,
valid_benchmark.test_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(test_y,
valid_benchmark.test_stream[0].dataset[:][1]))
def test_benchmark_with_validation_stream_fixed_size(self):
pattern_shape = (3, 32, 32)
# Definition of training experiences
# Experience 1
experience_1_x = torch.zeros(100, *pattern_shape)
experience_1_y = torch.zeros(100, dtype=torch.long)
# Experience 2
experience_2_x = torch.zeros(80, *pattern_shape)
experience_2_y = torch.ones(80, dtype=torch.long)
# Test experience
test_x = torch.zeros(50, *pattern_shape)
test_y = torch.zeros(50, dtype=torch.long)
initial_benchmark_instance = tensors_benchmark(
train_tensors=[(experience_1_x, experience_1_y),
(experience_2_x, experience_2_y)],
test_tensors=[(test_x, test_y)],
task_labels=[0, 0], # Task label of each train exp
complete_test_set_only=True)
valid_benchmark = benchmark_with_validation_stream(
initial_benchmark_instance, 20, shuffle=False)
self.assertEqual(2, len(valid_benchmark.train_stream))
self.assertEqual(2, len(valid_benchmark.valid_stream))
self.assertEqual(1, len(valid_benchmark.test_stream))
self.assertTrue(valid_benchmark.complete_test_set_only)
self.assertEqual(80, len(valid_benchmark.train_stream[0].dataset))
self.assertEqual(60, len(valid_benchmark.train_stream[1].dataset))
self.assertEqual(20, len(valid_benchmark.valid_stream[0].dataset))
self.assertEqual(20, len(valid_benchmark.valid_stream[1].dataset))
self.assertTrue(
torch.equal(experience_1_x[:80],
valid_benchmark.train_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(experience_2_x[:60],
valid_benchmark.train_stream[1].dataset[:][0]))
self.assertTrue(
torch.equal(experience_1_y[:80],
valid_benchmark.train_stream[0].dataset[:][1]))
self.assertTrue(
torch.equal(experience_2_y[:60],
valid_benchmark.train_stream[1].dataset[:][1]))
self.assertTrue(
torch.equal(experience_1_x[80:],
valid_benchmark.valid_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(experience_2_x[60:],
valid_benchmark.valid_stream[1].dataset[:][0]))
self.assertTrue(
torch.equal(experience_1_y[80:],
valid_benchmark.valid_stream[0].dataset[:][1]))
self.assertTrue(
torch.equal(experience_2_y[60:],
valid_benchmark.valid_stream[1].dataset[:][1]))
self.assertTrue(
torch.equal(test_x, valid_benchmark.test_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(test_y, valid_benchmark.test_stream[0].dataset[:][1]))
def test_benchmark_with_validation_stream_rel_size(self):
pattern_shape = (3, 32, 32)
# Definition of training experiences
# Experience 1
experience_1_x = torch.zeros(100, *pattern_shape)
experience_1_y = torch.zeros(100, dtype=torch.long)
# Experience 2
experience_2_x = torch.zeros(80, *pattern_shape)
experience_2_y = torch.ones(80, dtype=torch.long)
# Test experience
test_x = torch.zeros(50, *pattern_shape)
test_y = torch.zeros(50, dtype=torch.long)
initial_benchmark_instance = tensors_benchmark(
train_tensors=[
(experience_1_x, experience_1_y),
(experience_2_x, experience_2_y),
],
test_tensors=[(test_x, test_y)],
task_labels=[0, 0], # Task label of each train exp
complete_test_set_only=True,
)
valid_benchmark = benchmark_with_validation_stream(
initial_benchmark_instance, 0.2, shuffle=False)
expected_rel_1_valid = int(100 * 0.2)
expected_rel_1_train = 100 - expected_rel_1_valid
expected_rel_2_valid = int(80 * 0.2)
expected_rel_2_train = 80 - expected_rel_2_valid
self.assertEqual(2, len(valid_benchmark.train_stream))
self.assertEqual(2, len(valid_benchmark.valid_stream))
self.assertEqual(1, len(valid_benchmark.test_stream))
self.assertTrue(valid_benchmark.complete_test_set_only)
self.assertEqual(expected_rel_1_train,
len(valid_benchmark.train_stream[0].dataset))
self.assertEqual(expected_rel_2_train,
len(valid_benchmark.train_stream[1].dataset))
self.assertEqual(expected_rel_1_valid,
len(valid_benchmark.valid_stream[0].dataset))
self.assertEqual(expected_rel_2_valid,
len(valid_benchmark.valid_stream[1].dataset))
self.assertTrue(
torch.equal(
experience_1_x[:expected_rel_1_train],
valid_benchmark.train_stream[0].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_2_x[:expected_rel_2_train],
valid_benchmark.train_stream[1].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_1_y[:expected_rel_1_train],
valid_benchmark.train_stream[0].dataset[:][1],
))
self.assertTrue(
torch.equal(
experience_2_y[:expected_rel_2_train],
valid_benchmark.train_stream[1].dataset[:][1],
))
self.assertTrue(
torch.equal(
experience_1_x[expected_rel_1_train:],
valid_benchmark.valid_stream[0].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_2_x[expected_rel_2_train:],
valid_benchmark.valid_stream[1].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_1_y[expected_rel_1_train:],
valid_benchmark.valid_stream[0].dataset[:][1],
))
self.assertTrue(
torch.equal(
experience_2_y[expected_rel_2_train:],
valid_benchmark.valid_stream[1].dataset[:][1],
))
self.assertTrue(
torch.equal(test_x, valid_benchmark.test_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(test_y, valid_benchmark.test_stream[0].dataset[:][1]))
def test_lazy_benchmark_with_validation_stream_fixed_size(self):
pattern_shape = (3, 32, 32)
# Definition of training experiences
# Experience 1
experience_1_x = torch.zeros(100, *pattern_shape)
experience_1_y = torch.zeros(100, dtype=torch.long)
experience_1_dataset = AvalancheTensorDataset(experience_1_x,
experience_1_y)
# Experience 2
experience_2_x = torch.zeros(80, *pattern_shape)
experience_2_y = torch.ones(80, dtype=torch.long)
experience_2_dataset = AvalancheTensorDataset(experience_2_x,
experience_2_y)
# Test experience
test_x = torch.zeros(50, *pattern_shape)
test_y = torch.zeros(50, dtype=torch.long)
experience_test = AvalancheTensorDataset(test_x, test_y)
def train_gen():
# Lazy generator of the training stream
for dataset in [experience_1_dataset, experience_2_dataset]:
yield dataset
def test_gen():
# Lazy generator of the test stream
for dataset in [experience_test]:
yield dataset
initial_benchmark_instance = create_lazy_generic_benchmark(
train_generator=LazyStreamDefinition(train_gen(), 2, [0, 0]),
test_generator=LazyStreamDefinition(test_gen(), 1, [0]),
complete_test_set_only=True,
dataset_type=AvalancheDatasetType.CLASSIFICATION)
valid_benchmark = benchmark_with_validation_stream(
initial_benchmark_instance, 20, shuffle=False)
self.assertEqual(2, len(valid_benchmark.train_stream))
self.assertEqual(2, len(valid_benchmark.valid_stream))
self.assertEqual(1, len(valid_benchmark.test_stream))
self.assertTrue(valid_benchmark.complete_test_set_only)
self.assertEqual(80, len(valid_benchmark.train_stream[0].dataset))
self.assertEqual(60, len(valid_benchmark.train_stream[1].dataset))
self.assertEqual(20, len(valid_benchmark.valid_stream[0].dataset))
self.assertEqual(20, len(valid_benchmark.valid_stream[1].dataset))
self.assertTrue(
torch.equal(experience_1_x[:80],
valid_benchmark.train_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(experience_2_x[:60],
valid_benchmark.train_stream[1].dataset[:][0]))
self.assertTrue(
torch.equal(experience_1_y[:80],
valid_benchmark.train_stream[0].dataset[:][1]))
self.assertTrue(
torch.equal(experience_2_y[:60],
valid_benchmark.train_stream[1].dataset[:][1]))
self.assertTrue(
torch.equal(experience_1_x[80:],
valid_benchmark.valid_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(experience_2_x[60:],
valid_benchmark.valid_stream[1].dataset[:][0]))
self.assertTrue(
torch.equal(experience_1_y[80:],
valid_benchmark.valid_stream[0].dataset[:][1]))
self.assertTrue(
torch.equal(experience_2_y[60:],
valid_benchmark.valid_stream[1].dataset[:][1]))
self.assertTrue(
torch.equal(test_x, valid_benchmark.test_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(test_y, valid_benchmark.test_stream[0].dataset[:][1]))
