def test_tensors_benchmark(self):
pattern_shape = (3, 32, 32)
# Definition of training experiences
# Experience 1
experience_1_x = torch.zeros(100, *pattern_shape)
experience_1_y = torch.zeros(100, dtype=torch.long)
# Experience 2
experience_2_x = torch.zeros(80, *pattern_shape)
experience_2_y = torch.ones(80, dtype=torch.long)
# Test experience
test_x = torch.zeros(50, *pattern_shape)
test_y = torch.zeros(50, dtype=torch.long)
generic_scenario = tensors_benchmark(
train_tensors=[(experience_1_x, experience_1_y),
(experience_2_x, experience_2_y)],
test_tensors=[(test_x, test_y)],
task_labels=[0, 0], # Task label of each train exp
complete_test_set_only=True)
self.assertEqual(2, len(generic_scenario.train_stream))
self.assertEqual(1, len(generic_scenario.test_stream))
def test_tensor_benchmark_type(self):
n_experiences = 3
tensors_benchmark(
train_tensors=[(torch.randn(2, 3), torch.zeros(2))
for _ in range(n_experiences)],
test_tensors=[(torch.randn(2, 3), torch.zeros(2))
for _ in range(n_experiences)],
task_labels=[0] * n_experiences,
complete_test_set_only=False,
)
tensors_benchmark(
train_tensors=[(torch.randn(2, 3), torch.zeros(2))
for _ in range(n_experiences)],
test_tensors=[(torch.randn(2, 3), torch.zeros(2))],
task_labels=[0] * n_experiences,
complete_test_set_only=True,
)
with self.assertRaises(Exception):
tensors_benchmark(
train_tensors=[(torch.randn(2, 3), torch.zeros(2))
for _ in range(n_experiences)],
test_tensors=[(torch.randn(2, 3), torch.zeros(2))
for _ in range(n_experiences)],
task_labels=[0] * n_experiences,
complete_test_set_only=True,
)
def test_benchmark_with_validation_stream_rel_size(self):
pattern_shape = (3, 32, 32)
# Definition of training experiences
# Experience 1
experience_1_x = torch.zeros(100, *pattern_shape)
experience_1_y = torch.zeros(100, dtype=torch.long)
# Experience 2
experience_2_x = torch.zeros(80, *pattern_shape)
experience_2_y = torch.ones(80, dtype=torch.long)
# Test experience
test_x = torch.zeros(50, *pattern_shape)
test_y = torch.zeros(50, dtype=torch.long)
initial_benchmark_instance = tensors_benchmark(
train_tensors=[
(experience_1_x, experience_1_y),
(experience_2_x, experience_2_y),
],
test_tensors=[(test_x, test_y)],
task_labels=[0, 0], # Task label of each train exp
complete_test_set_only=True,
)
valid_benchmark = benchmark_with_validation_stream(
initial_benchmark_instance, 0.2, shuffle=False)
expected_rel_1_valid = int(100 * 0.2)
expected_rel_1_train = 100 - expected_rel_1_valid
expected_rel_2_valid = int(80 * 0.2)
expected_rel_2_train = 80 - expected_rel_2_valid
self.assertEqual(2, len(valid_benchmark.train_stream))
self.assertEqual(2, len(valid_benchmark.valid_stream))
self.assertEqual(1, len(valid_benchmark.test_stream))
self.assertTrue(valid_benchmark.complete_test_set_only)
self.assertEqual(expected_rel_1_train,
len(valid_benchmark.train_stream[0].dataset))
self.assertEqual(expected_rel_2_train,
len(valid_benchmark.train_stream[1].dataset))
self.assertEqual(expected_rel_1_valid,
len(valid_benchmark.valid_stream[0].dataset))
self.assertEqual(expected_rel_2_valid,
len(valid_benchmark.valid_stream[1].dataset))
self.assertTrue(
torch.equal(
experience_1_x[:expected_rel_1_train],
valid_benchmark.train_stream[0].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_2_x[:expected_rel_2_train],
valid_benchmark.train_stream[1].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_1_y[:expected_rel_1_train],
valid_benchmark.train_stream[0].dataset[:][1],
))
self.assertTrue(
torch.equal(
experience_2_y[:expected_rel_2_train],
valid_benchmark.train_stream[1].dataset[:][1],
))
self.assertTrue(
torch.equal(
experience_1_x[expected_rel_1_train:],
valid_benchmark.valid_stream[0].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_2_x[expected_rel_2_train:],
valid_benchmark.valid_stream[1].dataset[:][0],
))
self.assertTrue(
torch.equal(
experience_1_y[expected_rel_1_train:],
valid_benchmark.valid_stream[0].dataset[:][1],
))
self.assertTrue(
torch.equal(
experience_2_y[expected_rel_2_train:],
valid_benchmark.valid_stream[1].dataset[:][1],
))
self.assertTrue(
torch.equal(test_x, valid_benchmark.test_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(test_y, valid_benchmark.test_stream[0].dataset[:][1]))
def test_data_incremental_benchmark(self):
pattern_shape = (3, 32, 32)
# Definition of training experiences
# Experience 1
experience_1_x = torch.zeros(100, *pattern_shape)
experience_1_y = torch.zeros(100, dtype=torch.long)
# Experience 2
experience_2_x = torch.zeros(80, *pattern_shape)
experience_2_y = torch.ones(80, dtype=torch.long)
# Test experience
test_x = torch.zeros(50, *pattern_shape)
test_y = torch.zeros(50, dtype=torch.long)
initial_benchmark_instance = tensors_benchmark(
train_tensors=[
(experience_1_x, experience_1_y),
(experience_2_x, experience_2_y),
],
test_tensors=[(test_x, test_y)],
task_labels=[0, 0], # Task label of each train exp
complete_test_set_only=True,
)
data_incremental_instance = data_incremental_benchmark(
initial_benchmark_instance, 12, shuffle=False, drop_last=False)
self.assertEqual(16, len(data_incremental_instance.train_stream))
self.assertEqual(1, len(data_incremental_instance.test_stream))
self.assertTrue(data_incremental_instance.complete_test_set_only)
tensor_idx = 0
ref_tensor_x = experience_1_x
ref_tensor_y = experience_1_y
for exp in data_incremental_instance.train_stream:
if exp.current_experience == 8:
# Last mini-exp from 1st exp
self.assertEqual(4, len(exp.dataset))
elif exp.current_experience == 15:
# Last mini-exp from 2nd exp
self.assertEqual(8, len(exp.dataset))
else:
# Other mini-exp
self.assertEqual(12, len(exp.dataset))
if tensor_idx >= 100:
ref_tensor_x = experience_2_x
ref_tensor_y = experience_2_y
tensor_idx = 0
for x, y, *_ in exp.dataset:
self.assertTrue(torch.equal(ref_tensor_x[tensor_idx], x))
self.assertTrue(torch.equal(ref_tensor_y[tensor_idx], y))
tensor_idx += 1
exp = data_incremental_instance.test_stream[0]
self.assertEqual(50, len(exp.dataset))
tensor_idx = 0
for x, y, *_ in exp.dataset:
self.assertTrue(torch.equal(test_x[tensor_idx], x))
self.assertTrue(torch.equal(test_y[tensor_idx], y))
tensor_idx += 1
def test_benchmark_with_validation_stream_fixed_size(self):
pattern_shape = (3, 32, 32)
# Definition of training experiences
# Experience 1
experience_1_x = torch.zeros(100, *pattern_shape)
experience_1_y = torch.zeros(100, dtype=torch.long)
# Experience 2
experience_2_x = torch.zeros(80, *pattern_shape)
experience_2_y = torch.ones(80, dtype=torch.long)
# Test experience
test_x = torch.zeros(50, *pattern_shape)
test_y = torch.zeros(50, dtype=torch.long)
initial_benchmark_instance = tensors_benchmark(
train_tensors=[(experience_1_x, experience_1_y),
(experience_2_x, experience_2_y)],
test_tensors=[(test_x, test_y)],
task_labels=[0, 0], # Task label of each train exp
complete_test_set_only=True)
valid_benchmark = benchmark_with_validation_stream(
initial_benchmark_instance, 20, shuffle=False)
self.assertEqual(2, len(valid_benchmark.train_stream))
self.assertEqual(2, len(valid_benchmark.valid_stream))
self.assertEqual(1, len(valid_benchmark.test_stream))
self.assertTrue(valid_benchmark.complete_test_set_only)
self.assertEqual(80, len(valid_benchmark.train_stream[0].dataset))
self.assertEqual(60, len(valid_benchmark.train_stream[1].dataset))
self.assertEqual(20, len(valid_benchmark.valid_stream[0].dataset))
self.assertEqual(20, len(valid_benchmark.valid_stream[1].dataset))
self.assertTrue(
torch.equal(experience_1_x[:80],
valid_benchmark.train_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(experience_2_x[:60],
valid_benchmark.train_stream[1].dataset[:][0]))
self.assertTrue(
torch.equal(experience_1_y[:80],
valid_benchmark.train_stream[0].dataset[:][1]))
self.assertTrue(
torch.equal(experience_2_y[:60],
valid_benchmark.train_stream[1].dataset[:][1]))
self.assertTrue(
torch.equal(experience_1_x[80:],
valid_benchmark.valid_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(experience_2_x[60:],
valid_benchmark.valid_stream[1].dataset[:][0]))
self.assertTrue(
torch.equal(experience_1_y[80:],
valid_benchmark.valid_stream[0].dataset[:][1]))
self.assertTrue(
torch.equal(experience_2_y[60:],
valid_benchmark.valid_stream[1].dataset[:][1]))
self.assertTrue(
torch.equal(test_x, valid_benchmark.test_stream[0].dataset[:][0]))
self.assertTrue(
torch.equal(test_y, valid_benchmark.test_stream[0].dataset[:][1]))
