def test_sit_single_dataset_reproducibility_data(self):
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True,
download=True)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False,
download=True)
nc_scenario_ref = nc_scenario(mnist_train,
mnist_test,
5,
task_labels=False,
shuffle=True,
seed=5678)
my_nc_scenario = nc_scenario(
mnist_train,
mnist_test,
-1,
task_labels=False,
reproducibility_data=nc_scenario_ref.get_reproducibility_data())
self.assertEqual(nc_scenario_ref.train_exps_patterns_assignment,
my_nc_scenario.train_exps_patterns_assignment)
self.assertEqual(nc_scenario_ref.test_exps_patterns_assignment,
my_nc_scenario.test_exps_patterns_assignment)
def load_scenario(self, fast_test=False, use_task_labels=False):
"""
Returns a NC Scenario from a fake dataset of 10 classes, 5 experiences,
2 classes per experience.
:param fast_test: if True loads fake data, MNIST otherwise.
"""
if fast_test:
n_samples_per_class = 100
dataset = make_classification(n_samples=10 * n_samples_per_class,
n_classes=10,
n_features=6,
n_informative=6,
n_redundant=0)
X = torch.from_numpy(dataset[0]).float()
y = torch.from_numpy(dataset[1]).long()
train_X, test_X, train_y, test_y = train_test_split(X,
y,
train_size=0.6,
shuffle=True,
stratify=y)
train_dataset = TensorDataset(train_X, train_y)
test_dataset = TensorDataset(test_X, test_y)
my_nc_scenario = nc_scenario(train_dataset,
test_dataset,
5,
task_labels=use_task_labels)
else:
mnist_train = MNIST('./data/mnist',
train=True,
download=True,
transform=Compose([ToTensor()]))
mnist_test = MNIST('./data/mnist',
train=False,
download=True,
transform=Compose([ToTensor()]))
my_nc_scenario = nc_scenario(mnist_train,
mnist_test,
5,
task_labels=use_task_labels,
seed=1234)
return my_nc_scenario
def load_scenario(self, fast_test=False, use_task_labels=False):
"""
Returns a NC Scenario from a fake dataset of 10 classes, 5 experiences,
2 classes per experience.
:param fast_test: if True loads fake data, MNIST otherwise.
"""
if fast_test:
my_nc_scenario = get_fast_scenario()
else:
mnist_train = MNIST('./data/mnist',
train=True,
download=True,
transform=Compose([ToTensor()]))
mnist_test = MNIST('./data/mnist',
train=False,
download=True,
transform=Compose([ToTensor()]))
my_nc_scenario = nc_scenario(mnist_train,
mnist_test,
5,
task_labels=use_task_labels,
seed=1234)
return my_nc_scenario
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}"
if torch.cuda.is_available() and
args.cuda >= 0 else "cpu")
n_batches = 5
# ---------
# --- TRANSFORMATIONS
train_transform = transforms.Compose([
RandomCrop(28, padding=4),
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
test_transform = transforms.Compose([
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
# ---------
# --- SCENARIO CREATION
mnist_train = MNIST('./data/mnist', train=True,
download=True, transform=train_transform)
mnist_test = MNIST('./data/mnist', train=False,
download=True, transform=test_transform)
scenario = nc_scenario(
mnist_train, mnist_test, n_batches, task_labels=False, seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(
minibatch=True, epoch=True, experience=True, stream=True),
loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
ExperienceForgetting(),
loggers=[interactive_logger])
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(model, torch.optim.Adam(model.parameters(), lr=0.001),
CrossEntropyLoss(),
train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device,
plugins=[ReplayPlugin(mem_size=10000)],
evaluator=eval_plugin
)
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience ", experience.current_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def test_sit_single_dataset_fixed_order_subset(self):
order = [2, 5, 7, 8, 9, 0, 1, 4]
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True,
download=True)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False,
download=True)
my_nc_scenario = nc_scenario(mnist_train,
mnist_test,
4,
task_labels=True,
fixed_class_order=order,
class_ids_from_zero_in_each_exp=True)
self.assertEqual(4, len(my_nc_scenario.classes_in_experience))
all_classes = []
for task_id in range(4):
self.assertEqual(
2, len(my_nc_scenario.classes_in_experience[task_id]))
self.assertEqual(set(order[task_id * 2:(task_id + 1) * 2]),
my_nc_scenario.original_classes_in_exp[task_id])
all_classes.extend(my_nc_scenario.classes_in_experience[task_id])
self.assertEqual([0, 1] * 4, all_classes)
def test_sit_single_dataset_batch_size(self):
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True,
download=True)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False,
download=True)
my_nc_scenario = nc_scenario(mnist_train,
mnist_test,
3,
task_labels=False,
per_exp_classes={
0: 5,
2: 2
})
self.assertEqual(3, my_nc_scenario.n_experiences)
self.assertEqual(10, my_nc_scenario.n_classes)
all_classes = set()
for batch_id in range(3):
all_classes.update(my_nc_scenario.classes_in_experience[batch_id])
self.assertEqual(10, len(all_classes))
self.assertEqual(5, len(my_nc_scenario.classes_in_experience[0]))
self.assertEqual(3, len(my_nc_scenario.classes_in_experience[1]))
self.assertEqual(2, len(my_nc_scenario.classes_in_experience[2]))
def test_sit_single_dataset_remap_indexes(self):
order = [2, 3, 5, 8, 9, 1, 4, 6]
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True, download=True)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False, download=True)
my_nc_scenario = nc_scenario(
mnist_train, mnist_test, 4, task_labels=False,
fixed_class_order=order, class_ids_from_zero_from_first_exp=True)
self.assertEqual(4, len(my_nc_scenario.classes_in_experience))
all_classes = []
for batch_id in range(4):
self.assertEqual(
2, len(my_nc_scenario.classes_in_experience[batch_id]))
all_classes.extend(my_nc_scenario.classes_in_experience[batch_id])
self.assertEqual(list(range(8)), all_classes)
# Regression test for issue #258
for i, experience in enumerate(my_nc_scenario.train_stream):
unique_dataset_classes = sorted(set(experience.dataset.targets))
expected_dataset_classes = list(range(2 * i, 2 * (i+1)))
self.assertListEqual(expected_dataset_classes,
unique_dataset_classes)
self.assertListEqual(
sorted(order[2 * i:2 * (i+1)]),
sorted(my_nc_scenario.original_classes_in_exp[i]))
def test_sit_single_dataset(self):
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True,
download=True)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False,
download=True)
my_nc_scenario = nc_scenario(mnist_train,
mnist_test,
5,
task_labels=False,
shuffle=True,
seed=1234)
self.assertEqual(5, my_nc_scenario.n_experiences)
self.assertEqual(10, my_nc_scenario.n_classes)
for batch_id in range(my_nc_scenario.n_experiences):
self.assertEqual(
2, len(my_nc_scenario.classes_in_experience[batch_id]))
all_classes = set()
for batch_id in range(5):
all_classes.update(my_nc_scenario.classes_in_experience[batch_id])
self.assertEqual(10, len(all_classes))
def load_ar1_scenario(self):
"""
Returns a NC Scenario from a fake dataset of 10 classes, 5 experiences,
2 classes per experience. This toy scenario is intended
"""
n_samples_per_class = 50
dataset = make_classification(n_samples=10 * n_samples_per_class,
n_classes=10,
n_features=224 * 224 * 3,
n_informative=6,
n_redundant=0)
X = torch.from_numpy(dataset[0]).reshape(-1, 3, 224, 224).float()
y = torch.from_numpy(dataset[1]).long()
train_X, test_X, train_y, test_y = train_test_split(X,
y,
train_size=0.6,
shuffle=True,
stratify=y)
train_dataset = TensorDataset(train_X, train_y)
test_dataset = TensorDataset(test_X, test_y)
my_nc_scenario = nc_scenario(train_dataset,
test_dataset,
5,
task_labels=False)
return my_nc_scenario
def get_fast_scenario(use_task_labels=False, shuffle=True):
n_samples_per_class = 100
dataset = make_classification(n_samples=10 * n_samples_per_class,
n_classes=10,
n_features=6,
n_informative=6,
n_redundant=0)
X = torch.from_numpy(dataset[0]).float()
y = torch.from_numpy(dataset[1]).long()
train_X, test_X, train_y, test_y = train_test_split(X,
y,
train_size=0.6,
shuffle=True,
stratify=y)
train_dataset = TensorDataset(train_X, train_y)
test_dataset = TensorDataset(test_X, test_y)
my_nc_scenario = nc_scenario(train_dataset,
test_dataset,
5,
task_labels=use_task_labels,
shuffle=shuffle)
return my_nc_scenario
def test_mt_single_dataset(self):
mnist_train = MNIST('./data/mnist', train=True, download=True)
mnist_test = MNIST('./data/mnist', train=False, download=True)
my_nc_scenario = nc_scenario(mnist_train,
mnist_test,
5,
task_labels=True,
shuffle=True,
seed=1234,
class_ids_from_zero_in_each_exp=True)
self.assertEqual(5, my_nc_scenario.n_experiences)
self.assertEqual(10, my_nc_scenario.n_classes)
for task_id in range(5):
self.assertEqual(
2, len(my_nc_scenario.classes_in_experience[task_id]))
all_classes = set()
all_original_classes = set()
for task_id in range(5):
all_classes.update(my_nc_scenario.classes_in_experience[task_id])
all_original_classes.update(
my_nc_scenario.original_classes_in_exp[task_id])
self.assertEqual(2, len(all_classes))
self.assertEqual(10, len(all_original_classes))
def create_scenario(self, task_labels=False):
n_samples_per_class = 20
dataset = make_classification(n_samples=10 * n_samples_per_class,
n_classes=10,
n_features=6,
n_informative=6,
n_redundant=0)
X = torch.from_numpy(dataset[0]).float()
y = torch.from_numpy(dataset[1]).long()
train_X, test_X, train_y, test_y = train_test_split(X,
y,
train_size=0.6,
shuffle=True,
stratify=y)
train_dataset = TensorDataset(train_X, train_y)
test_dataset = TensorDataset(test_X, test_y)
return nc_scenario(train_dataset,
test_dataset,
5,
task_labels=task_labels,
fixed_class_order=list(range(10)))
def test_sit_multi_dataset_merge(self):
split_mapping = [0, 1, 2, 3, 4, 0, 1, 2, 3, 4]
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True, download=True)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False, download=True)
train_part1 = make_nc_transformation_subset(
mnist_train, None, None, range(5))
train_part2 = make_nc_transformation_subset(
mnist_train, None, None, range(5, 10))
train_part2 = AvalancheSubset(
train_part2, class_mapping=split_mapping)
test_part1 = make_nc_transformation_subset(
mnist_test, None, None, range(5))
test_part2 = make_nc_transformation_subset(
mnist_test, None, None, range(5, 10))
test_part2 = AvalancheSubset(test_part2,
class_mapping=split_mapping)
my_nc_scenario = nc_scenario(
[train_part1, train_part2], [test_part1, test_part2], 5,
task_labels=False, shuffle=True, seed=1234)
self.assertEqual(5, my_nc_scenario.n_experiences)
self.assertEqual(10, my_nc_scenario.n_classes)
for batch_id in range(5):
self.assertEqual(
2, len(my_nc_scenario.classes_in_experience[batch_id]))
all_classes = set()
for batch_id in range(5):
all_classes.update(my_nc_scenario.classes_in_experience[batch_id])
self.assertEqual(10, len(all_classes))
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}"
if torch.cuda.is_available() and
args.cuda >= 0 else "cpu")
# ---------
# --- TRANSFORMATIONS
train_transform = transforms.Compose([
RandomCrop(28, padding=4),
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
test_transform = transforms.Compose([
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
# ---------
# --- SCENARIO CREATION
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True, download=True, transform=train_transform)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False, download=True, transform=test_transform)
scenario = nc_scenario(
mnist_train, mnist_test, 5, task_labels=False, seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
eval_plugin = EvaluationPlugin(
accuracy_metrics(epoch=True, experience=True, stream=True),
loss_metrics(epoch=True, experience=True, stream=True),
# save image should be False to appropriately view
# results in Interactive Logger.
# a tensor will be printed
StreamConfusionMatrix(save_image=False, normalize='all'),
loggers=InteractiveLogger()
)
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(
model, SGD(model.parameters(), lr=0.001, momentum=0.9),
CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100,
device=device, evaluator=eval_plugin, plugins=[ReplayPlugin(5000)])
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience: ", experience.current_experience)
print("Current Classes: ", experience.classes_in_this_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
# ---------
# --- TRANSFORMATIONS
train_transform = transforms.Compose([
RandomCrop(28, padding=4),
ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
test_transform = transforms.Compose(
[ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))])
# ---------
# --- SCENARIO CREATION
mnist_train = MNIST('./data/mnist',
train=True,
download=True,
transform=train_transform)
mnist_test = MNIST('./data/mnist',
train=False,
download=True,
transform=test_transform)
scenario = nc_scenario(mnist_train,
mnist_test,
5,
task_labels=False,
seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(model,
SGD(model.parameters(), lr=0.001, momentum=0.9),
CrossEntropyLoss(),
train_mb_size=100,
train_epochs=4,
eval_mb_size=100,
device=device)
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience: ", experience.current_experience)
print("Current Classes: ", experience.classes_in_this_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def test_sit_single_dataset_remap_indexes_each_exp(self):
order = [2, 3, 5, 8, 9, 1, 4, 6]
mnist_train = MNIST('./data/mnist', train=True, download=True)
mnist_test = MNIST('./data/mnist', train=False, download=True)
with self.assertRaises(ValueError):
# class_ids_from_zero_* are mutually exclusive
nc_scenario(mnist_train,
mnist_test,
4,
task_labels=False,
fixed_class_order=order,
class_ids_from_zero_from_first_exp=True,
class_ids_from_zero_in_each_exp=True)
my_nc_scenario = nc_scenario(mnist_train,
mnist_test,
4,
task_labels=False,
fixed_class_order=order,
class_ids_from_zero_in_each_exp=True)
self.assertEqual(4, len(my_nc_scenario.classes_in_experience))
all_classes = []
for batch_id in range(4):
self.assertEqual(
2, len(my_nc_scenario.classes_in_experience[batch_id]))
all_classes.extend(my_nc_scenario.classes_in_experience[batch_id])
self.assertEqual(8, len(all_classes))
self.assertListEqual([0, 1], sorted(set(all_classes)))
# Regression test for issue #258
for i, experience in enumerate(my_nc_scenario.train_stream):
unique_dataset_classes = sorted(set(experience.dataset.targets))
expected_dataset_classes = [0, 1]
self.assertListEqual(expected_dataset_classes,
unique_dataset_classes)
self.assertListEqual(
sorted(order[2 * i:2 * (i + 1)]),
sorted(my_nc_scenario.original_classes_in_exp[i]))
def test_mt_single_dataset_fixed_order(self):
order = [2, 3, 5, 7, 8, 9, 0, 1, 4, 6]
mnist_train = MNIST('./data/mnist', train=True, download=True)
mnist_test = MNIST('./data/mnist', train=False, download=True)
my_nc_scenario = nc_scenario(
mnist_train, mnist_test, 5, task_labels=True,
fixed_class_order=order, class_ids_from_zero_in_each_exp=False)
all_classes = []
for task_id in range(5):
all_classes.extend(my_nc_scenario.classes_in_experience[task_id])
self.assertEqual(order, all_classes)
def load_ar1_scenario(self, fast_test=False):
"""
Returns a NC Scenario from a fake dataset of 10 classes, 5 experiences,
2 classes per experience. This toy scenario is intended
:param fast_test: if True loads fake data, MNIST otherwise.
"""
if fast_test:
n_samples_per_class = 50
dataset = make_classification(n_samples=10 * n_samples_per_class,
n_classes=10,
n_features=224 * 224 * 3,
n_informative=6,
n_redundant=0)
X = torch.from_numpy(dataset[0]).reshape(-1, 3, 224, 224).float()
y = torch.from_numpy(dataset[1]).long()
train_X, test_X, train_y, test_y = train_test_split(X,
y,
train_size=0.6,
shuffle=True,
stratify=y)
train_dataset = TensorDataset(train_X, train_y)
test_dataset = TensorDataset(test_X, test_y)
my_nc_scenario = nc_scenario(train_dataset,
test_dataset,
5,
task_labels=False)
else:
train_transform = transforms.Compose([
Resize(224),
ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
test_transform = transforms.Compose([
Resize(224),
ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
my_nc_scenario = SplitCIFAR10(5,
train_transform=train_transform,
eval_transform=test_transform)
return my_nc_scenario
def test_sit_single_dataset_fixed_order(self):
order = [2, 3, 5, 7, 8, 9, 0, 1, 4, 6]
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True, download=True)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False, download=True)
my_nc_scenario = nc_scenario(
mnist_train, mnist_test, 5, task_labels=False,
fixed_class_order=order)
all_classes = []
for batch_id in range(5):
all_classes.extend(my_nc_scenario.classes_in_experience[batch_id])
self.assertEqual(order, all_classes)
def test_sit_single_dataset_fixed_subset_no_remap_idx(self):
order = [2, 5, 7, 8, 9, 0, 1, 4]
mnist_train = MNIST('./data/mnist', train=True, download=True)
mnist_test = MNIST('./data/mnist', train=False, download=True)
my_nc_scenario = nc_scenario(
mnist_train, mnist_test, 2, task_labels=True,
fixed_class_order=order, class_ids_from_zero_in_each_exp=False)
self.assertEqual(2, len(my_nc_scenario.classes_in_experience))
all_classes = set()
for task_id in range(2):
self.assertEqual(4, len(my_nc_scenario.classes_in_experience[task_id]))
self.assertEqual(set(order[task_id*4:(task_id+1)*4]),
my_nc_scenario.original_classes_in_exp[task_id])
all_classes.update(my_nc_scenario.classes_in_experience[task_id])
self.assertEqual(set(order), all_classes)
def test_nc_sit_slicing(self):
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True,
download=True)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False,
download=True)
my_nc_scenario = nc_scenario(mnist_train,
mnist_test,
5,
task_labels=False,
shuffle=True,
seed=1234)
experience: NCExperience
for batch_id, experience in enumerate(my_nc_scenario.train_stream):
self.assertEqual(batch_id, experience.current_experience)
self.assertIsInstance(experience, NCExperience)
for batch_id, experience in enumerate(my_nc_scenario.test_stream):
self.assertEqual(batch_id, experience.current_experience)
self.assertIsInstance(experience, NCExperience)
iterable_slice = [3, 4, 1]
sliced_stream = my_nc_scenario.train_stream[iterable_slice]
self.assertIsInstance(sliced_stream, GenericScenarioStream)
self.assertEqual(len(iterable_slice), len(sliced_stream))
self.assertEqual('train', sliced_stream.name)
for batch_id, experience in enumerate(sliced_stream):
self.assertEqual(iterable_slice[batch_id],
experience.current_experience)
self.assertIsInstance(experience, NCExperience)
sliced_stream = my_nc_scenario.test_stream[iterable_slice]
self.assertIsInstance(sliced_stream, GenericScenarioStream)
self.assertEqual(len(iterable_slice), len(sliced_stream))
self.assertEqual('test', sliced_stream.name)
for batch_id, experience in enumerate(sliced_stream):
self.assertEqual(iterable_slice[batch_id],
experience.current_experience)
self.assertIsInstance(experience, NCExperience)
def test_mt_single_dataset_task_size(self):
mnist_train = MNIST('./data/mnist', train=True, download=True)
mnist_test = MNIST('./data/mnist', train=False, download=True)
my_nc_scenario = nc_scenario(
mnist_train, mnist_test, 3, task_labels=True,
per_exp_classes={0: 5, 2: 2},
class_ids_from_zero_in_each_exp=True)
self.assertEqual(3, my_nc_scenario.n_experiences)
self.assertEqual(10, my_nc_scenario.n_classes)
all_classes = set()
for task_id in range(3):
all_classes.update(my_nc_scenario.classes_in_experience[task_id])
self.assertEqual(5, len(all_classes))
self.assertEqual(5, len(my_nc_scenario.classes_in_experience[0]))
self.assertEqual(3, len(my_nc_scenario.classes_in_experience[1]))
self.assertEqual(2, len(my_nc_scenario.classes_in_experience[2]))
def test_sit_single_dataset_fixed_order_subset(self):
order = [2, 3, 5, 8, 9, 1, 4, 6]
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True, download=True)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False, download=True)
my_nc_scenario = nc_scenario(
mnist_train, mnist_test, 4, task_labels=False,
fixed_class_order=order)
self.assertEqual(4, len(my_nc_scenario.classes_in_experience))
all_classes = set()
for batch_id in range(4):
self.assertEqual(
2, len(my_nc_scenario.classes_in_experience[batch_id]))
all_classes.update(my_nc_scenario.classes_in_experience[batch_id])
self.assertEqual(set(order), all_classes)
def test_sit_multi_dataset_one_batch_per_set(self):
split_mapping = [0, 1, 2, 0, 1, 2, 3, 4, 5, 6]
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True,
download=True)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False,
download=True)
train_part1 = make_nc_transformation_subset(mnist_train, None, None,
range(3))
train_part2 = make_nc_transformation_subset(mnist_train, None, None,
range(3, 10))
train_part2 = AvalancheSubset(train_part2, class_mapping=split_mapping)
test_part1 = make_nc_transformation_subset(mnist_test, None, None,
range(3))
test_part2 = make_nc_transformation_subset(mnist_test, None, None,
range(3, 10))
test_part2 = AvalancheSubset(test_part2, class_mapping=split_mapping)
my_nc_scenario = nc_scenario([train_part1, train_part2],
[test_part1, test_part2],
2,
task_labels=False,
shuffle=True,
seed=1234,
one_dataset_per_exp=True)
self.assertEqual(2, my_nc_scenario.n_experiences)
self.assertEqual(10, my_nc_scenario.n_classes)
all_classes = set()
for batch_id in range(2):
all_classes.update(my_nc_scenario.classes_in_experience[batch_id])
self.assertEqual(10, len(all_classes))
self.assertTrue(
(my_nc_scenario.classes_in_experience[0] == {0, 1, 2}
and my_nc_scenario.classes_in_experience[1] == set(range(3, 10)))
or (my_nc_scenario.classes_in_experience[0] == set(range(3, 10))
and my_nc_scenario.classes_in_experience[1] == {0, 1, 2}))
def load_scenario(use_task_labels=False, fast_test=True):
"""
Returns a NC Scenario from a fake dataset of 10 classes, 5 experiences,
2 classes per experience.
"""
if fast_test:
my_nc_scenario = get_fast_scenario(use_task_labels)
else:
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True,
download=True,
transform=Compose([ToTensor()]))
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False,
download=True,
transform=Compose([ToTensor()]))
my_nc_scenario = nc_scenario(mnist_train,
mnist_test,
5,
task_labels=use_task_labels,
seed=1234)
return my_nc_scenario
def SplitOmniglot(n_experiences: int,
return_task_id=False,
seed: Optional[int] = None,
fixed_class_order: Optional[Sequence[int]] = None,
train_transform=_default_omniglot_train_transform,
eval_transform=_default_omniglot_eval_transform):
"""
Creates a CL scenario using the OMNIGLOT dataset.
If the dataset is not present in the computer, this method will
automatically download and store it.
The returned scenario will return experiences containing all patterns of a
subset of classes, which means that each class is only seen "once".
This is one of the most common scenarios in the Continual Learning
literature. Common names used in literature to describe this kind of
scenario are "Class Incremental", "New Classes", etc.
By default, an equal amount of classes will be assigned to each experience.
OMNIGLOT consists of 1623 classes, which means that the number of
experiences can be 1, 3, 541, 1623.
This generator doesn't force a choice on the availability of task labels,
a choice that is left to the user (see the `return_task_id` parameter for
more info on task labels).
The scenario instance returned by this method will have two fields,
`train_stream` and `test_stream`, which can be iterated to obtain
training and test :class:`Experience`. Each Experience contains the
`dataset` and the associated task label.
The scenario API is quite simple and is uniform across all scenario
generators. It is recommended to check the tutorial of the "benchmark" API,
which contains usage examples ranging from "basic" to "advanced".
:param n_experiences: The number of incremental experiences in the current
scenario. The value of this parameter should be a divisor of 10.
:param return_task_id: if True, a progressive task id is returned for every
experience. If False, all experiences will have a task ID of 0.
:param seed: A valid int used to initialize the random number generator.
Can be None.
:param fixed_class_order: A list of class IDs used to define the class
order. If None, value of ``seed`` will be used to define the class
order. If non-None, ``seed`` parameter will be ignored.
Defaults to None.
:param train_transform: The transformation to apply to the training data,
e.g. a random crop, a normalization or a concatenation of different
transformations (see torchvision.transform documentation for a
comprehensive list of possible transformations).
If no transformation is passed, the default train transformation
will be used.
:param eval_transform: The transformation to apply to the test data,
e.g. a random crop, a normalization or a concatenation of different
transformations (see torchvision.transform documentation for a
comprehensive list of possible transformations).
If no transformation is passed, the default test transformation
will be used.
:returns: A properly initialized :class:`NCScenario` instance.
"""
omniglot_train, omniglot_test = _get_omniglot_dataset(
train_transform, eval_transform)
if return_task_id:
return nc_scenario(train_dataset=omniglot_train,
test_dataset=omniglot_test,
n_experiences=n_experiences,
task_labels=True,
seed=seed,
fixed_class_order=fixed_class_order,
class_ids_from_zero_in_each_exp=True)
else:
return nc_scenario(train_dataset=omniglot_train,
test_dataset=omniglot_test,
n_experiences=n_experiences,
task_labels=False,
seed=seed,
fixed_class_order=fixed_class_order)
def RotatedOmniglot(
n_experiences: int,
seed: Optional[int] = None,
rotations_list: Optional[Sequence[int]] = None,
train_transform=_default_omniglot_train_transform,
eval_transform=_default_omniglot_eval_transform) -> NCScenario:
"""
Creates a Rotated Omniglot scenario.
If the dataset is not present in the computer, this method will
automatically download and store it.
Random angles are used to rotate the Omniglot images in ``n_experiences``
different manners. This means that each experience is
composed of all the original 1623 Omniglot classes, but each image is
rotated in a different way.
The scenario instance returned by this method will have two fields,
`train_stream` and `test_stream`, which can be iterated to obtain
training and test :class:`Experience`. Each Experience contains the
`dataset` and the associated task label.
A progressive task label, starting from "0", is applied to each experience.
The scenario API is quite simple and is uniform across all scenario
generators. It is recommended to check the tutorial of the "benchmark" API,
which contains usage examples ranging from "basic" to "advanced".
:param n_experiences: The number of experiences (tasks) in the current
scenario. It indicates how many different rotations of the Omniglot
dataset have to be created.
:param seed: A valid int used to initialize the random number generator.
Can be None.
:param rotations_list: A list of rotations values in degrees (from -180 to
180) used to define the rotations. The rotation specified in position
0 of the list will be applied to the task 0, the rotation specified in
position 1 will be applied to task 1 and so on.
If None, value of ``seed`` will be used to define the rotations.
If non-None, ``seed`` parameter will be ignored.
Defaults to None.
:param train_transform: The transformation to apply to the training data
after the random rotation, e.g. a random crop, a normalization or a
concatenation of different transformations (see torchvision.transform
documentation for a comprehensive list of possible transformations).
If no transformation is passed, the default train transformation
will be used.
:param eval_transform: The transformation to apply to the test data
after the random rotation, e.g. a random crop, a normalization or a
concatenation of different transformations (see torchvision.transform
documentation for a comprehensive list of possible transformations).
If no transformation is passed, the default test transformation
will be used.
:returns: A properly initialized :class:`NCScenario` instance.
"""
if rotations_list is None:
rng_rotate = np.random.RandomState(seed)
rotations_list = [
rng_rotate.randint(-180, 181) for _ in range(n_experiences)
]
else:
assert len(rotations_list) == n_experiences, "The number of rotations" \
" should match the number" \
" of incremental experiences."
assert all(-180 <= rotations_list[i] <= 180
for i in range(len(rotations_list))), "The value of a rotation" \
" should be between -180" \
" and 180 degrees."
list_train_dataset = []
list_test_dataset = []
# for every incremental experience
for experience in range(n_experiences):
rotation_angle = rotations_list[experience]
rotation = RandomRotation(degrees=(rotation_angle, rotation_angle))
omniglot_train, omniglot_test = _get_omniglot_dataset(
rotation, rotation)
# Freeze the rotation, then add the user defined transformations
rotated_train = omniglot_train \
.freeze_transforms() \
.add_transforms(train_transform)
rotated_test = omniglot_test \
.freeze_transforms() \
.add_transforms(eval_transform)
list_train_dataset.append(rotated_train)
list_test_dataset.append(rotated_test)
return nc_scenario(list_train_dataset,
list_test_dataset,
n_experiences=len(list_train_dataset),
task_labels=True,
shuffle=False,
class_ids_from_zero_in_each_exp=True,
one_dataset_per_exp=True)
def PermutedOmniglot(
n_experiences: int,
seed: Optional[int] = None,
train_transform: Any = _default_omniglot_train_transform,
eval_transform: Any = _default_omniglot_eval_transform) -> NCScenario:
"""
Creates a Permuted Omniglot scenario.
If the dataset is not present in the computer, this method will
automatically download and store it.
Random pixel permutations are used to permute the Omniglot images in
``n_experiences`` different manners. This means that each experience is
composed of all the original 1623 Omniglot classes, but the pixel in the
images are permuted in a different way.
The scenario instance returned by this method will have two fields,
`train_stream` and `test_stream`, which can be iterated to obtain
training and test :class:`Experience`. Each Experience contains the
`dataset` and the associated task label.
A progressive task label, starting from "0", is applied to each experience.
The scenario API is quite simple and is uniform across all scenario
generators. It is recommended to check the tutorial of the "benchmark" API,
which contains usage examples ranging from "basic" to "advanced".
:param n_experiences: The number of experiences (tasks) in the current
scenario. It indicates how many different permutations of the Omniglot
dataset have to be created.
:param seed: A valid int used to initialize the random number generator.
Can be None.
:param train_transform: The transformation to apply to the training data
before the random permutation, e.g. a random crop, a normalization or a
concatenation of different transformations (see torchvision.transform
documentation for a comprehensive list of possible transformations).
If no transformation is passed, the default train transformation
will be used.
:param eval_transform: The transformation to apply to the test data
before the random permutation, e.g. a random crop, a normalization or a
concatenation of different transformations (see torchvision.transform
documentation for a comprehensive list of possible transformations).
If no transformation is passed, the default test transformation
will be used.
:returns: A properly initialized :class:`NCScenario` instance.
"""
list_train_dataset = []
list_test_dataset = []
rng_permute = np.random.RandomState(seed)
# for every incremental experience
for _ in range(n_experiences):
# choose a random permutation of the pixels in the image
idx_permute = torch.from_numpy(rng_permute.permutation(784)).type(
torch.int64)
permutation = PixelsPermutation(idx_permute)
omniglot_train, omniglot_test = _get_omniglot_dataset(
permutation, permutation)
# Freeze the permutation, then add the user defined transformations
permuted_train = omniglot_train \
.freeze_transforms() \
.add_transforms(train_transform)
permuted_test = omniglot_test \
.freeze_transforms() \
.add_transforms(eval_transform)
list_train_dataset.append(permuted_train)
list_test_dataset.append(permuted_test)
return nc_scenario(list_train_dataset,
list_test_dataset,
n_experiences=len(list_train_dataset),
task_labels=True,
shuffle=False,
class_ids_from_zero_in_each_exp=True,
one_dataset_per_exp=True)
def SplitImageNet(root,
n_experiences=10,
per_exp_classes=None,
return_task_id=False,
seed=0,
fixed_class_order=None,
train_transform=_default_train_transform,
test_transform=_default_test_transform):
"""
Creates a CL scenario using the Tiny ImageNet dataset.
If the dataset is not present in the computer the method automatically
download it and store the data in the data folder.
:param root: Base path where Imagenet data are stored.
:param n_experiences: The number of experience in the current scenario.
:param per_exp_classes: Is not None, a dictionary whose keys are
(0-indexed) experience IDs and their values are the number of classes
to include in the respective experiences. The dictionary doesn't
have to contain a key for each experience! All the remaining exps
will contain an equal amount of the remaining classes. The
remaining number of classes must be divisible without remainder
by the remaining number of experiences. For instance,
if you want to include 50 classes in the first experience
while equally distributing remaining classes across remaining
experiences, just pass the "{0: 50}" dictionary as the
per_experience_classes parameter. Defaults to None.
:param return_task_id: if True, a progressive task id is returned for every
experience. If False, all experiences will have a task ID of 0.
:param seed: A valid int used to initialize the random number generator.
Can be None.
:param fixed_class_order: A list of class IDs used to define the class
order. If None, value of ``seed`` will be used to define the class
order. If non-None, ``seed`` parameter will be ignored.
Defaults to None.
:param train_transform: The transformation to apply to the training data,
e.g. a random crop, a normalization or a concatenation of different
transformations (see torchvision.transform documentation for a
comprehensive list of possible transformations).
If no transformation is passed, the default train transformation
will be used.
:param test_transform: The transformation to apply to the test data,
e.g. a random crop, a normalization or a concatenation of different
transformations (see torchvision.transform documentation for a
comprehensive list of possible transformations).
If no transformation is passed, the default test transformation
will be used.
:returns: A :class:`NCMultiTaskScenario` instance initialized for the the
MT scenario using CIFAR10 if the parameter ``return_task_id`` is True,
a :class:`NCSingleTaskScenario` initialized for the SIT scenario using
CIFAR10 otherwise.
"""
train_set, test_set = _get_imagenet_dataset(root, train_transform,
test_transform)
if return_task_id:
return nc_scenario(train_dataset=train_set,
test_dataset=test_set,
n_experiences=n_experiences,
task_labels=True,
per_exp_classes=per_exp_classes,
seed=seed,
fixed_class_order=fixed_class_order,
class_ids_from_zero_in_each_exp=True)
else:
return nc_scenario(train_dataset=train_set,
test_dataset=test_set,
n_experiences=n_experiences,
task_labels=False,
per_exp_classes=per_exp_classes,
seed=seed,
fixed_class_order=fixed_class_order)
def test_mt_multi_dataset_one_task_per_set(self):
split_mapping = [0, 1, 2, 0, 1, 2, 3, 4, 5, 6]
mnist_train = MNIST('./data/mnist', train=True, download=True)
mnist_test = MNIST('./data/mnist', train=False, download=True)
train_part1 = make_nc_transformation_subset(mnist_train, None, None,
range(3))
train_part2 = make_nc_transformation_subset(mnist_train, None, None,
range(3, 10))
train_part2 = AvalancheSubset(train_part2, class_mapping=split_mapping)
test_part1 = make_nc_transformation_subset(mnist_test, None, None,
range(3))
test_part2 = make_nc_transformation_subset(mnist_test, None, None,
range(3, 10))
test_part2 = AvalancheSubset(test_part2, class_mapping=split_mapping)
my_nc_scenario = nc_scenario([train_part1, train_part2],
[test_part1, test_part2],
2,
task_labels=True,
seed=1234,
class_ids_from_zero_in_each_exp=True,
one_dataset_per_exp=True)
self.assertEqual(2, my_nc_scenario.n_experiences)
self.assertEqual(10, my_nc_scenario.n_classes)
self.assertEqual(2, len(my_nc_scenario.train_stream))
self.assertEqual(2, len(my_nc_scenario.test_stream))
exp_classes_train = []
exp_classes_test = []
all_classes_train = set()
all_classes_test = set()
task_info: NCExperience
for task_id, task_info in enumerate(my_nc_scenario.train_stream):
self.assertLessEqual(task_id, 1)
all_classes_train.update(
my_nc_scenario.classes_in_experience[task_id])
exp_classes_train.append(task_info.classes_in_this_experience)
self.assertEqual(7, len(all_classes_train))
for task_id, task_info in enumerate(my_nc_scenario.test_stream):
self.assertLessEqual(task_id, 1)
all_classes_test.update(
my_nc_scenario.classes_in_experience[task_id])
exp_classes_test.append(task_info.classes_in_this_experience)
self.assertEqual(7, len(all_classes_test))
self.assertTrue(
(my_nc_scenario.classes_in_experience[0] == {0, 1, 2}
and my_nc_scenario.classes_in_experience[1] == set(range(0, 7)))
or (my_nc_scenario.classes_in_experience[0] == set(range(0, 7))
and my_nc_scenario.classes_in_experience[1] == {0, 1, 2}))
exp_classes_ref1 = [list(range(3)), list(range(7))]
exp_classes_ref2 = [list(range(7)), list(range(3))]
self.assertTrue(exp_classes_train == exp_classes_ref1
or exp_classes_train == exp_classes_ref2)
if exp_classes_train == exp_classes_ref1:
self.assertTrue(exp_classes_test == exp_classes_ref1)
else:
self.assertTrue(exp_classes_test == exp_classes_ref2)
