def test_classes_in_this_experience(self):
train_exps = []
tensor_x = torch.rand(200, 3, 28, 28)
tensor_y = torch.randint(0, 70, (200, ))
tensor_t = torch.randint(0, 5, (200, ))
train_exps.append(
AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t))
tensor_x = torch.rand(200, 3, 28, 28)
tensor_y = torch.randint(0, 100, (200, ))
tensor_t = torch.randint(0, 5, (200, ))
train_exps.append(
AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t))
test_exps = []
test_x = torch.rand(200, 3, 28, 28)
test_y = torch.randint(100, 200, (200, ))
test_t = torch.randint(0, 5, (200, ))
test_exps.append(
AvalancheTensorDataset(test_x, test_y, task_labels=test_t))
other_stream_exps = []
other_x = torch.rand(200, 3, 28, 28)
other_y = torch.randint(400, 600, (200, ))
other_t = torch.randint(0, 5, (200, ))
other_stream_exps.append(
AvalancheTensorDataset(other_x, other_y, task_labels=other_t))
benchmark_instance = dataset_benchmark(
train_datasets=train_exps,
test_datasets=test_exps,
other_streams_datasets={'other': other_stream_exps})
train_exp_0: GenericExperience = benchmark_instance.train_stream[0]
train_exp_1: GenericExperience = benchmark_instance.train_stream[1]
train_0_classes = train_exp_0.classes_in_this_experience
train_1_classes = train_exp_1.classes_in_this_experience
train_0_classes_min = min(train_0_classes)
train_1_classes_min = min(train_1_classes)
train_0_classes_max = max(train_0_classes)
train_1_classes_max = max(train_1_classes)
self.assertGreaterEqual(train_0_classes_min, 0)
self.assertLess(train_0_classes_max, 70)
self.assertGreaterEqual(train_1_classes_min, 0)
self.assertLess(train_1_classes_max, 100)
test_exp_0: GenericExperience = benchmark_instance.test_stream[0]
test_0_classes = test_exp_0.classes_in_this_experience
test_0_classes_min = min(test_0_classes)
test_0_classes_max = max(test_0_classes)
self.assertGreaterEqual(test_0_classes_min, 100)
self.assertLess(test_0_classes_max, 200)
other_exp_0: GenericExperience = benchmark_instance.other_stream[0]
other_0_classes = other_exp_0.classes_in_this_experience
other_0_classes_min = min(other_0_classes)
other_0_classes_max = max(other_0_classes)
self.assertGreaterEqual(other_0_classes_min, 400)
self.assertLess(other_0_classes_max, 600)
def test_dataset_benchmark_avalanche_dataset(self):
train_MNIST = AvalancheDataset(MNIST('./data/mnist',
train=True,
download=True),
task_labels=0)
test_MNIST = AvalancheDataset(MNIST('./data/mnist',
train=False,
download=True),
task_labels=0)
train_cifar10 = AvalancheDataset(CIFAR10('./data/cifar10',
train=True,
download=True),
task_labels=1)
test_cifar10 = AvalancheDataset(CIFAR10('./data/cifar10',
train=False,
download=True),
task_labels=1)
generic_scenario = dataset_benchmark([train_MNIST, train_cifar10],
[test_MNIST, test_cifar10])
self.assertEqual(0, generic_scenario.train_stream[0].task_label)
self.assertEqual(1, generic_scenario.train_stream[1].task_label)
self.assertEqual(0, generic_scenario.test_stream[0].task_label)
self.assertEqual(1, generic_scenario.test_stream[1].task_label)
def test_dataset_benchmark_avalanche_dataset(self):
train_MNIST = AvalancheDataset(MNIST(root=expanduser("~") +
"/.avalanche/data/mnist/",
train=True,
download=True),
task_labels=0)
test_MNIST = AvalancheDataset(MNIST(root=expanduser("~") +
"/.avalanche/data/mnist/",
train=False,
download=True),
task_labels=0)
train_cifar10 = AvalancheDataset(CIFAR10(root=expanduser("~") +
"/.avalanche/data/cifar10/",
train=True,
download=True),
task_labels=1)
test_cifar10 = AvalancheDataset(CIFAR10(root=expanduser("~") +
"/.avalanche/data/cifar10/",
train=False,
download=True),
task_labels=1)
generic_benchmark = dataset_benchmark([train_MNIST, train_cifar10],
[test_MNIST, test_cifar10])
self.assertEqual(0, generic_benchmark.train_stream[0].task_label)
self.assertEqual(1, generic_benchmark.train_stream[1].task_label)
self.assertEqual(0, generic_benchmark.test_stream[0].task_label)
self.assertEqual(1, generic_benchmark.test_stream[1].task_label)
def test_dataset_benchmark(self):
train_MNIST = MNIST('./data/mnist', train=True, download=True)
test_MNIST = MNIST('./data/mnist', train=False, download=True)
train_cifar10 = CIFAR10('./data/cifar10', train=True, download=True)
test_cifar10 = CIFAR10('./data/cifar10', train=False, download=True)
generic_scenario = dataset_benchmark([train_MNIST, train_cifar10],
[test_MNIST, test_cifar10])
def test_dataset_benchmark(self):
train_MNIST = MNIST(root=default_dataset_location("mnist"),
train=True,
download=True)
test_MNIST = MNIST(root=default_dataset_location("mnist"),
train=False,
download=True)
train_cifar10 = CIFAR10(root=default_dataset_location("cifar10"),
train=True,
download=True)
test_cifar10 = CIFAR10(root=default_dataset_location("cifar10"),
train=False,
download=True)
generic_benchmark = dataset_benchmark([train_MNIST, train_cifar10],
[test_MNIST, test_cifar10])
def test_dataset_benchmark_avalanche_dataset(self):
train_MNIST = AvalancheDataset(
MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
),
task_labels=0,
)
test_MNIST = AvalancheDataset(
MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
),
task_labels=0,
)
train_cifar10 = AvalancheDataset(
CIFAR10(
root=default_dataset_location("cifar10"),
train=True,
download=True,
),
task_labels=1,
)
test_cifar10 = AvalancheDataset(
CIFAR10(
root=default_dataset_location("cifar10"),
train=False,
download=True,
),
task_labels=1,
)
generic_benchmark = dataset_benchmark([train_MNIST, train_cifar10],
[test_MNIST, test_cifar10])
self.assertEqual(0, generic_benchmark.train_stream[0].task_label)
self.assertEqual(1, generic_benchmark.train_stream[1].task_label)
self.assertEqual(0, generic_benchmark.test_stream[0].task_label)
self.assertEqual(1, generic_benchmark.test_stream[1].task_label)
def test_dataset_benchmark(self):
train_MNIST = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True,
download=True)
test_MNIST = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False,
download=True)
train_cifar10 = CIFAR10(root=expanduser("~") +
"/.avalanche/data/cifar10/",
train=True,
download=True)
test_cifar10 = CIFAR10(root=expanduser("~") +
"/.avalanche/data/cifar10/",
train=False,
download=True)
generic_scenario = dataset_benchmark([train_MNIST, train_cifar10],
[test_MNIST, test_cifar10])
def test_classes_in_exp(self):
train_exps = []
tensor_x = torch.rand(200, 3, 28, 28)
tensor_y = torch.randint(0, 70, (200, ))
tensor_t = torch.randint(0, 5, (200, ))
train_exps.append(
AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t))
tensor_x = torch.rand(200, 3, 28, 28)
tensor_y = torch.randint(0, 100, (200, ))
tensor_t = torch.randint(0, 5, (200, ))
train_exps.append(
AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t))
test_exps = []
test_x = torch.rand(200, 3, 28, 28)
test_y = torch.randint(100, 200, (200, ))
test_t = torch.randint(0, 5, (200, ))
test_exps.append(
AvalancheTensorDataset(test_x, test_y, task_labels=test_t))
other_stream_exps = []
other_x = torch.rand(200, 3, 28, 28)
other_y = torch.randint(400, 600, (200, ))
other_t = torch.randint(0, 5, (200, ))
other_stream_exps.append(
AvalancheTensorDataset(other_x, other_y, task_labels=other_t))
benchmark_instance = dataset_benchmark(
train_datasets=train_exps,
test_datasets=test_exps,
other_streams_datasets={'other': other_stream_exps})
train_0_classes = benchmark_instance.classes_in_experience['train'][0]
train_1_classes = benchmark_instance.classes_in_experience['train'][1]
train_0_classes_min = min(train_0_classes)
train_1_classes_min = min(train_1_classes)
train_0_classes_max = max(train_0_classes)
train_1_classes_max = max(train_1_classes)
self.assertGreaterEqual(train_0_classes_min, 0)
self.assertLess(train_0_classes_max, 70)
self.assertGreaterEqual(train_1_classes_min, 0)
self.assertLess(train_1_classes_max, 100)
# Test deprecated behavior
train_0_classes = benchmark_instance.classes_in_experience[0]
train_1_classes = benchmark_instance.classes_in_experience[1]
train_0_classes_min = min(train_0_classes)
train_1_classes_min = min(train_1_classes)
train_0_classes_max = max(train_0_classes)
train_1_classes_max = max(train_1_classes)
self.assertGreaterEqual(train_0_classes_min, 0)
self.assertLess(train_0_classes_max, 70)
self.assertGreaterEqual(train_1_classes_min, 0)
self.assertLess(train_1_classes_max, 100)
# End test deprecated behavior
test_0_classes = benchmark_instance.classes_in_experience['test'][0]
test_0_classes_min = min(test_0_classes)
test_0_classes_max = max(test_0_classes)
self.assertGreaterEqual(test_0_classes_min, 100)
self.assertLess(test_0_classes_max, 200)
other_0_classes = benchmark_instance.classes_in_experience['other'][0]
other_0_classes_min = min(other_0_classes)
other_0_classes_max = max(other_0_classes)
self.assertGreaterEqual(other_0_classes_min, 400)
self.assertLess(other_0_classes_max, 600)
def setUpClass(cls) -> None:
torch.manual_seed(0)
np.random.seed(0)
random.seed(0)
n_samples_per_class = 100
datasets = []
for i in range(3):
dataset = make_classification(n_samples=3 * n_samples_per_class,
n_classes=3,
n_features=3,
n_informative=3,
n_redundant=0)
X = torch.from_numpy(dataset[0]).float()
y = torch.from_numpy(dataset[1]).long()
train_X, test_X, train_y, test_y = train_test_split(X,
y,
train_size=0.5,
shuffle=True,
stratify=y)
datasets.append((train_X, train_y, test_X, test_y))
tr_ds = [
AvalancheTensorDataset(
tr_X,
tr_y,
dataset_type=AvalancheDatasetType.CLASSIFICATION,
task_labels=torch.randint(0, 3, (150, )).tolist())
for tr_X, tr_y, _, _ in datasets
]
ts_ds = [
AvalancheTensorDataset(
ts_X,
ts_y,
dataset_type=AvalancheDatasetType.CLASSIFICATION,
task_labels=torch.randint(0, 3, (150, )).tolist())
for _, _, ts_X, ts_y in datasets
]
benchmark = dataset_benchmark(train_datasets=tr_ds,
test_datasets=ts_ds)
model = SimpleMLP(num_classes=3, input_size=3)
f = open('log.txt', 'w')
text_logger = TextLogger(f)
eval_plugin = EvaluationPlugin(
accuracy_metrics(minibatch=True,
epoch=True,
epoch_running=True,
experience=True,
stream=True,
trained_experience=True),
loss_metrics(minibatch=True,
epoch=True,
epoch_running=True,
experience=True,
stream=True),
forgetting_metrics(experience=True, stream=True),
confusion_matrix_metrics(num_classes=3,
save_image=False,
normalize='all',
stream=True),
bwt_metrics(experience=True, stream=True),
forward_transfer_metrics(experience=True, stream=True),
cpu_usage_metrics(minibatch=True,
epoch=True,
epoch_running=True,
experience=True,
stream=True),
timing_metrics(minibatch=True,
epoch=True,
epoch_running=True,
experience=True,
stream=True),
ram_usage_metrics(every=0.5,
minibatch=True,
epoch=True,
experience=True,
stream=True),
disk_usage_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
MAC_metrics(minibatch=True, epoch=True, experience=True),
loggers=[text_logger],
collect_all=True) # collect all metrics (set to True by default)
cl_strategy = BaseStrategy(model,
SGD(model.parameters(),
lr=0.001,
momentum=0.9),
CrossEntropyLoss(),
train_mb_size=2,
train_epochs=2,
eval_mb_size=2,
device=DEVICE,
evaluator=eval_plugin,
eval_every=1)
for i, experience in enumerate(benchmark.train_stream):
cl_strategy.train(experience,
eval_streams=[benchmark.test_stream],
shuffle=False)
cl_strategy.eval(benchmark.test_stream)
cls.all_metrics = cl_strategy.evaluator.get_all_metrics()
f.close()
# # Uncomment me to regenerate the reference metrics. Make sure
# # the old tests were passing for all unchanged metrics
# with open(os.path.join(pathlib.Path(__file__).parent.absolute(),
# 'target_metrics',
# 'tpp.pickle'), 'wb') as f:
# pickle.dump(dict(cls.all_metrics), f,
# protocol=4)
with open(
os.path.join(
pathlib.Path(__file__).parent.absolute(), 'target_metrics',
'tpp.pickle'), 'rb') as f:
cls.ref = pickle.load(f)
def CTrL(
stream_name: str,
save_to_disk: bool = False,
path: Path = default_dataset_location(""),
seed: int = None,
n_tasks: int = None,
):
"""
Gives access to the Continual Transfer Learning benchmark streams
introduced in https://arxiv.org/abs/2012.12631.
:param stream_name: Name of the test stream to generate. Must be one of
`s_plus`, `s_minus`, `s_in`, `s_out` and `s_pl`.
:param save_to_disk: Whether to save each stream on the disk or load
everything in memory. Setting it to `True` will save memory but takes more
time on the first generation using the corresponding seed.
:param path: The path under which the generated stream will be saved if
save_to_disk is True.
:param seed: The seed to use to generate the streams. If no seed is given,
a random one will be used to make sure that the generated stream can
be reproduced.
:param n_tasks: The number of tasks to generate. This parameter is only
relevant for the `s_long` stream, as all other streams have a fixed number
of tasks.
:return: A scenario containing 3 streams: train, val and test.
"""
seed = seed or random.randint(0, sys.maxsize)
if stream_name != "s_long" and n_tasks is not None:
raise ValueError("The n_tasks parameter can only be used with the "
f'"s_long" stream, asked {n_tasks} for {stream_name}')
elif stream_name == "s_long" and n_tasks is None:
n_tasks = 100
stream = ctrl.get_stream(stream_name, seed)
if save_to_disk:
folder = path / "ctrl" / stream_name / f"seed_{seed}"
# Train, val and test experiences
exps = [[], [], []]
for t_id, t in enumerate(tqdm(stream, desc=f"Loading {stream_name}"), ):
trans = transforms.Normalize(t.statistics["mean"], t.statistics["std"])
for split, split_name, exp in zip(t.datasets, t.split_names, exps):
samples, labels = split.tensors
task_labels = [t.id] * samples.size(0)
if save_to_disk:
exp_folder = folder / f"exp_{t_id}" / split_name
exp_folder.mkdir(parents=True, exist_ok=True)
files = []
for i, (sample, label) in enumerate(zip(samples, labels)):
sample_path = exp_folder / f"sample_{i}.png"
if not sample_path.exists():
F.to_pil_image(sample).save(sample_path)
files.append((sample_path, label.item()))
common_root, exp_paths_list = common_paths_root(files)
paths_dataset = PathsDataset(common_root, exp_paths_list)
dataset = AvalancheDataset(
paths_dataset,
task_labels=task_labels,
transform=transforms.Compose(
[transforms.ToTensor(), trans]),
)
else:
dataset = AvalancheTensorDataset(
samples,
labels.squeeze(1),
task_labels=task_labels,
transform=trans,
)
exp.append(dataset)
if stream_name == "s_long" and t_id == n_tasks - 1:
break
return dataset_benchmark(
train_datasets=exps[0],
test_datasets=exps[2],
other_streams_datasets=dict(val=exps[1]),
)
