def test_classes_in_this_experience(self): train_exps = [] tensor_x = torch.rand(200, 3, 28, 28) tensor_y = torch.randint(0, 70, (200, )) tensor_t = torch.randint(0, 5, (200, )) train_exps.append( AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t)) tensor_x = torch.rand(200, 3, 28, 28) tensor_y = torch.randint(0, 100, (200, )) tensor_t = torch.randint(0, 5, (200, )) train_exps.append( AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t)) test_exps = [] test_x = torch.rand(200, 3, 28, 28) test_y = torch.randint(100, 200, (200, )) test_t = torch.randint(0, 5, (200, )) test_exps.append( AvalancheTensorDataset(test_x, test_y, task_labels=test_t)) other_stream_exps = [] other_x = torch.rand(200, 3, 28, 28) other_y = torch.randint(400, 600, (200, )) other_t = torch.randint(0, 5, (200, )) other_stream_exps.append( AvalancheTensorDataset(other_x, other_y, task_labels=other_t)) benchmark_instance = dataset_benchmark( train_datasets=train_exps, test_datasets=test_exps, other_streams_datasets={'other': other_stream_exps}) train_exp_0: GenericExperience = benchmark_instance.train_stream[0] train_exp_1: GenericExperience = benchmark_instance.train_stream[1] train_0_classes = train_exp_0.classes_in_this_experience train_1_classes = train_exp_1.classes_in_this_experience train_0_classes_min = min(train_0_classes) train_1_classes_min = min(train_1_classes) train_0_classes_max = max(train_0_classes) train_1_classes_max = max(train_1_classes) self.assertGreaterEqual(train_0_classes_min, 0) self.assertLess(train_0_classes_max, 70) self.assertGreaterEqual(train_1_classes_min, 0) self.assertLess(train_1_classes_max, 100) test_exp_0: GenericExperience = benchmark_instance.test_stream[0] test_0_classes = test_exp_0.classes_in_this_experience test_0_classes_min = min(test_0_classes) test_0_classes_max = max(test_0_classes) self.assertGreaterEqual(test_0_classes_min, 100) self.assertLess(test_0_classes_max, 200) other_exp_0: GenericExperience = benchmark_instance.other_stream[0] other_0_classes = other_exp_0.classes_in_this_experience other_0_classes_min = min(other_0_classes) other_0_classes_max = max(other_0_classes) self.assertGreaterEqual(other_0_classes_min, 400) self.assertLess(other_0_classes_max, 600)
def test_dataset_benchmark_avalanche_dataset(self): train_MNIST = AvalancheDataset(MNIST('./data/mnist', train=True, download=True), task_labels=0) test_MNIST = AvalancheDataset(MNIST('./data/mnist', train=False, download=True), task_labels=0) train_cifar10 = AvalancheDataset(CIFAR10('./data/cifar10', train=True, download=True), task_labels=1) test_cifar10 = AvalancheDataset(CIFAR10('./data/cifar10', train=False, download=True), task_labels=1) generic_scenario = dataset_benchmark([train_MNIST, train_cifar10], [test_MNIST, test_cifar10]) self.assertEqual(0, generic_scenario.train_stream[0].task_label) self.assertEqual(1, generic_scenario.train_stream[1].task_label) self.assertEqual(0, generic_scenario.test_stream[0].task_label) self.assertEqual(1, generic_scenario.test_stream[1].task_label)
def test_dataset_benchmark_avalanche_dataset(self): train_MNIST = AvalancheDataset(MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True), task_labels=0) test_MNIST = AvalancheDataset(MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True), task_labels=0) train_cifar10 = AvalancheDataset(CIFAR10(root=expanduser("~") + "/.avalanche/data/cifar10/", train=True, download=True), task_labels=1) test_cifar10 = AvalancheDataset(CIFAR10(root=expanduser("~") + "/.avalanche/data/cifar10/", train=False, download=True), task_labels=1) generic_benchmark = dataset_benchmark([train_MNIST, train_cifar10], [test_MNIST, test_cifar10]) self.assertEqual(0, generic_benchmark.train_stream[0].task_label) self.assertEqual(1, generic_benchmark.train_stream[1].task_label) self.assertEqual(0, generic_benchmark.test_stream[0].task_label) self.assertEqual(1, generic_benchmark.test_stream[1].task_label)
def test_dataset_benchmark(self): train_MNIST = MNIST('./data/mnist', train=True, download=True) test_MNIST = MNIST('./data/mnist', train=False, download=True) train_cifar10 = CIFAR10('./data/cifar10', train=True, download=True) test_cifar10 = CIFAR10('./data/cifar10', train=False, download=True) generic_scenario = dataset_benchmark([train_MNIST, train_cifar10], [test_MNIST, test_cifar10])
def test_dataset_benchmark(self): train_MNIST = MNIST(root=default_dataset_location("mnist"), train=True, download=True) test_MNIST = MNIST(root=default_dataset_location("mnist"), train=False, download=True) train_cifar10 = CIFAR10(root=default_dataset_location("cifar10"), train=True, download=True) test_cifar10 = CIFAR10(root=default_dataset_location("cifar10"), train=False, download=True) generic_benchmark = dataset_benchmark([train_MNIST, train_cifar10], [test_MNIST, test_cifar10])
def test_dataset_benchmark_avalanche_dataset(self): train_MNIST = AvalancheDataset( MNIST( root=default_dataset_location("mnist"), train=True, download=True, ), task_labels=0, ) test_MNIST = AvalancheDataset( MNIST( root=default_dataset_location("mnist"), train=False, download=True, ), task_labels=0, ) train_cifar10 = AvalancheDataset( CIFAR10( root=default_dataset_location("cifar10"), train=True, download=True, ), task_labels=1, ) test_cifar10 = AvalancheDataset( CIFAR10( root=default_dataset_location("cifar10"), train=False, download=True, ), task_labels=1, ) generic_benchmark = dataset_benchmark([train_MNIST, train_cifar10], [test_MNIST, test_cifar10]) self.assertEqual(0, generic_benchmark.train_stream[0].task_label) self.assertEqual(1, generic_benchmark.train_stream[1].task_label) self.assertEqual(0, generic_benchmark.test_stream[0].task_label) self.assertEqual(1, generic_benchmark.test_stream[1].task_label)
def test_dataset_benchmark(self): train_MNIST = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True) test_MNIST = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True) train_cifar10 = CIFAR10(root=expanduser("~") + "/.avalanche/data/cifar10/", train=True, download=True) test_cifar10 = CIFAR10(root=expanduser("~") + "/.avalanche/data/cifar10/", train=False, download=True) generic_scenario = dataset_benchmark([train_MNIST, train_cifar10], [test_MNIST, test_cifar10])
def test_classes_in_exp(self): train_exps = [] tensor_x = torch.rand(200, 3, 28, 28) tensor_y = torch.randint(0, 70, (200, )) tensor_t = torch.randint(0, 5, (200, )) train_exps.append( AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t)) tensor_x = torch.rand(200, 3, 28, 28) tensor_y = torch.randint(0, 100, (200, )) tensor_t = torch.randint(0, 5, (200, )) train_exps.append( AvalancheTensorDataset(tensor_x, tensor_y, task_labels=tensor_t)) test_exps = [] test_x = torch.rand(200, 3, 28, 28) test_y = torch.randint(100, 200, (200, )) test_t = torch.randint(0, 5, (200, )) test_exps.append( AvalancheTensorDataset(test_x, test_y, task_labels=test_t)) other_stream_exps = [] other_x = torch.rand(200, 3, 28, 28) other_y = torch.randint(400, 600, (200, )) other_t = torch.randint(0, 5, (200, )) other_stream_exps.append( AvalancheTensorDataset(other_x, other_y, task_labels=other_t)) benchmark_instance = dataset_benchmark( train_datasets=train_exps, test_datasets=test_exps, other_streams_datasets={'other': other_stream_exps}) train_0_classes = benchmark_instance.classes_in_experience['train'][0] train_1_classes = benchmark_instance.classes_in_experience['train'][1] train_0_classes_min = min(train_0_classes) train_1_classes_min = min(train_1_classes) train_0_classes_max = max(train_0_classes) train_1_classes_max = max(train_1_classes) self.assertGreaterEqual(train_0_classes_min, 0) self.assertLess(train_0_classes_max, 70) self.assertGreaterEqual(train_1_classes_min, 0) self.assertLess(train_1_classes_max, 100) # Test deprecated behavior train_0_classes = benchmark_instance.classes_in_experience[0] train_1_classes = benchmark_instance.classes_in_experience[1] train_0_classes_min = min(train_0_classes) train_1_classes_min = min(train_1_classes) train_0_classes_max = max(train_0_classes) train_1_classes_max = max(train_1_classes) self.assertGreaterEqual(train_0_classes_min, 0) self.assertLess(train_0_classes_max, 70) self.assertGreaterEqual(train_1_classes_min, 0) self.assertLess(train_1_classes_max, 100) # End test deprecated behavior test_0_classes = benchmark_instance.classes_in_experience['test'][0] test_0_classes_min = min(test_0_classes) test_0_classes_max = max(test_0_classes) self.assertGreaterEqual(test_0_classes_min, 100) self.assertLess(test_0_classes_max, 200) other_0_classes = benchmark_instance.classes_in_experience['other'][0] other_0_classes_min = min(other_0_classes) other_0_classes_max = max(other_0_classes) self.assertGreaterEqual(other_0_classes_min, 400) self.assertLess(other_0_classes_max, 600)
def setUpClass(cls) -> None: torch.manual_seed(0) np.random.seed(0) random.seed(0) n_samples_per_class = 100 datasets = [] for i in range(3): dataset = make_classification(n_samples=3 * n_samples_per_class, n_classes=3, n_features=3, n_informative=3, n_redundant=0) X = torch.from_numpy(dataset[0]).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split(X, y, train_size=0.5, shuffle=True, stratify=y) datasets.append((train_X, train_y, test_X, test_y)) tr_ds = [ AvalancheTensorDataset( tr_X, tr_y, dataset_type=AvalancheDatasetType.CLASSIFICATION, task_labels=torch.randint(0, 3, (150, )).tolist()) for tr_X, tr_y, _, _ in datasets ] ts_ds = [ AvalancheTensorDataset( ts_X, ts_y, dataset_type=AvalancheDatasetType.CLASSIFICATION, task_labels=torch.randint(0, 3, (150, )).tolist()) for _, _, ts_X, ts_y in datasets ] benchmark = dataset_benchmark(train_datasets=tr_ds, test_datasets=ts_ds) model = SimpleMLP(num_classes=3, input_size=3) f = open('log.txt', 'w') text_logger = TextLogger(f) eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, trained_experience=True), loss_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), forgetting_metrics(experience=True, stream=True), confusion_matrix_metrics(num_classes=3, save_image=False, normalize='all', stream=True), bwt_metrics(experience=True, stream=True), forward_transfer_metrics(experience=True, stream=True), cpu_usage_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), ram_usage_metrics(every=0.5, minibatch=True, epoch=True, experience=True, stream=True), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[text_logger], collect_all=True) # collect all metrics (set to True by default) cl_strategy = BaseStrategy(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=2, train_epochs=2, eval_mb_size=2, device=DEVICE, evaluator=eval_plugin, eval_every=1) for i, experience in enumerate(benchmark.train_stream): cl_strategy.train(experience, eval_streams=[benchmark.test_stream], shuffle=False) cl_strategy.eval(benchmark.test_stream) cls.all_metrics = cl_strategy.evaluator.get_all_metrics() f.close() # # Uncomment me to regenerate the reference metrics. Make sure # # the old tests were passing for all unchanged metrics # with open(os.path.join(pathlib.Path(__file__).parent.absolute(), # 'target_metrics', # 'tpp.pickle'), 'wb') as f: # pickle.dump(dict(cls.all_metrics), f, # protocol=4) with open( os.path.join( pathlib.Path(__file__).parent.absolute(), 'target_metrics', 'tpp.pickle'), 'rb') as f: cls.ref = pickle.load(f)
def CTrL( stream_name: str, save_to_disk: bool = False, path: Path = default_dataset_location(""), seed: int = None, n_tasks: int = None, ): """ Gives access to the Continual Transfer Learning benchmark streams introduced in https://arxiv.org/abs/2012.12631. :param stream_name: Name of the test stream to generate. Must be one of `s_plus`, `s_minus`, `s_in`, `s_out` and `s_pl`. :param save_to_disk: Whether to save each stream on the disk or load everything in memory. Setting it to `True` will save memory but takes more time on the first generation using the corresponding seed. :param path: The path under which the generated stream will be saved if save_to_disk is True. :param seed: The seed to use to generate the streams. If no seed is given, a random one will be used to make sure that the generated stream can be reproduced. :param n_tasks: The number of tasks to generate. This parameter is only relevant for the `s_long` stream, as all other streams have a fixed number of tasks. :return: A scenario containing 3 streams: train, val and test. """ seed = seed or random.randint(0, sys.maxsize) if stream_name != "s_long" and n_tasks is not None: raise ValueError("The n_tasks parameter can only be used with the " f'"s_long" stream, asked {n_tasks} for {stream_name}') elif stream_name == "s_long" and n_tasks is None: n_tasks = 100 stream = ctrl.get_stream(stream_name, seed) if save_to_disk: folder = path / "ctrl" / stream_name / f"seed_{seed}" # Train, val and test experiences exps = [[], [], []] for t_id, t in enumerate(tqdm(stream, desc=f"Loading {stream_name}"), ): trans = transforms.Normalize(t.statistics["mean"], t.statistics["std"]) for split, split_name, exp in zip(t.datasets, t.split_names, exps): samples, labels = split.tensors task_labels = [t.id] * samples.size(0) if save_to_disk: exp_folder = folder / f"exp_{t_id}" / split_name exp_folder.mkdir(parents=True, exist_ok=True) files = [] for i, (sample, label) in enumerate(zip(samples, labels)): sample_path = exp_folder / f"sample_{i}.png" if not sample_path.exists(): F.to_pil_image(sample).save(sample_path) files.append((sample_path, label.item())) common_root, exp_paths_list = common_paths_root(files) paths_dataset = PathsDataset(common_root, exp_paths_list) dataset = AvalancheDataset( paths_dataset, task_labels=task_labels, transform=transforms.Compose( [transforms.ToTensor(), trans]), ) else: dataset = AvalancheTensorDataset( samples, labels.squeeze(1), task_labels=task_labels, transform=trans, ) exp.append(dataset) if stream_name == "s_long" and t_id == n_tasks - 1: break return dataset_benchmark( train_datasets=exps[0], test_datasets=exps[2], other_streams_datasets=dict(val=exps[1]), )