def test_mt_single_dataset_reproducibility_data(self): mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True) nc_benchmark_ref = nc_benchmark(mnist_train, mnist_test, 5, task_labels=True, shuffle=True, seed=5678) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, -1, task_labels=True, reproducibility_data=nc_benchmark_ref.get_reproducibility_data()) self.assertEqual(nc_benchmark_ref.train_exps_patterns_assignment, my_nc_benchmark.train_exps_patterns_assignment) self.assertEqual(nc_benchmark_ref.test_exps_patterns_assignment, my_nc_benchmark.test_exps_patterns_assignment)
def test_sit_single_dataset_remap_indexes_each_exp(self): order = [2, 3, 5, 8, 9, 1, 4, 6] mnist_train = MNIST( root=default_dataset_location("mnist"), train=True, download=True, ) mnist_test = MNIST( root=default_dataset_location("mnist"), train=False, download=True, ) with self.assertRaises(ValueError): # class_ids_from_zero_* are mutually exclusive nc_benchmark( mnist_train, mnist_test, 4, task_labels=False, fixed_class_order=order, class_ids_from_zero_from_first_exp=True, class_ids_from_zero_in_each_exp=True, ) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, 4, task_labels=False, fixed_class_order=order, class_ids_from_zero_in_each_exp=True, ) self.assertEqual(4, len(my_nc_benchmark.classes_in_experience["train"])) all_classes = [] for batch_id in range(4): self.assertEqual( 2, len(my_nc_benchmark.classes_in_experience["train"][batch_id]) ) all_classes.extend( my_nc_benchmark.classes_in_experience["train"][batch_id] ) self.assertEqual(8, len(all_classes)) self.assertListEqual([0, 1], sorted(set(all_classes))) # Regression test for issue #258 for i, experience in enumerate(my_nc_benchmark.train_stream): unique_dataset_classes = sorted(set(experience.dataset.targets)) expected_dataset_classes = [0, 1] self.assertListEqual( expected_dataset_classes, unique_dataset_classes ) self.assertListEqual( sorted(order[2 * i : 2 * (i + 1)]), sorted(my_nc_benchmark.original_classes_in_exp[i]), )
def test_nc_benchmark_classes_in_exp_range(self): train_set = CIFAR100( default_dataset_location("cifar100"), train=True, download=True ) test_set = CIFAR100( default_dataset_location("cifar100"), train=False, download=True ) benchmark_instance = nc_benchmark( train_dataset=train_set, test_dataset=test_set, n_experiences=5, task_labels=False, seed=1234, shuffle=False, ) cie_data = benchmark_instance.classes_in_exp_range(0, None) self.assertEqual(5, len(cie_data)) for i in range(5): expected = set(range(i * 20, (i + 1) * 20)) self.assertSetEqual(expected, set(cie_data[i])) cie_data = benchmark_instance.classes_in_exp_range(1, 4) self.assertEqual(3, len(cie_data)) for i in range(1, 3): expected = set(range(i * 20, (i + 1) * 20)) self.assertSetEqual(expected, set(cie_data[i - 1])) random_class_order = list(range(100)) random.shuffle(random_class_order) benchmark_instance = nc_benchmark( train_dataset=train_set, test_dataset=test_set, n_experiences=5, task_labels=False, seed=1234, fixed_class_order=random_class_order, shuffle=False, ) cie_data = benchmark_instance.classes_in_exp_range(0, None) self.assertEqual(5, len(cie_data)) for i in range(5): expected = set(random_class_order[i * 20 : (i + 1) * 20]) self.assertSetEqual(expected, set(cie_data[i]))
def create_benchmark(task_labels=False, seed=None): n_samples_per_class = 20 dataset = make_classification( n_samples=10 * n_samples_per_class, n_classes=10, n_features=6, n_informative=6, n_redundant=0, random_state=seed, ) X = torch.from_numpy(dataset[0]).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split( X, y, train_size=0.6, shuffle=True, stratify=y, random_state=seed ) train_dataset = TensorDataset(train_X, train_y) test_dataset = TensorDataset(test_X, test_y) return nc_benchmark( train_dataset, test_dataset, 5, task_labels=task_labels, fixed_class_order=list(range(10)), )
def test_sit_single_dataset_fixed_order_subset(self): order = [2, 5, 7, 8, 9, 0, 1, 4] mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True) my_nc_benchmark = nc_benchmark(mnist_train, mnist_test, 4, task_labels=True, fixed_class_order=order, class_ids_from_zero_in_each_exp=True) self.assertEqual(4, len(my_nc_benchmark.classes_in_experience['train'])) all_classes = [] for task_id in range(4): self.assertEqual( 2, len(my_nc_benchmark.classes_in_experience['train'][task_id])) self.assertEqual(set(order[task_id * 2:(task_id + 1) * 2]), my_nc_benchmark.original_classes_in_exp[task_id]) all_classes.extend( my_nc_benchmark.classes_in_experience['train'][task_id]) self.assertEqual([0, 1] * 4, all_classes)
def test_mt_single_dataset_task_size(self): mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True) my_nc_benchmark = nc_benchmark(mnist_train, mnist_test, 3, task_labels=True, per_exp_classes={ 0: 5, 2: 2 }, class_ids_from_zero_in_each_exp=True) self.assertEqual(3, my_nc_benchmark.n_experiences) self.assertEqual(10, my_nc_benchmark.n_classes) all_classes = set() for task_id in range(3): all_classes.update( my_nc_benchmark.classes_in_experience['train'][task_id]) self.assertEqual(5, len(all_classes)) self.assertEqual( 5, len(my_nc_benchmark.classes_in_experience['train'][0])) self.assertEqual( 3, len(my_nc_benchmark.classes_in_experience['train'][1])) self.assertEqual( 2, len(my_nc_benchmark.classes_in_experience['train'][2]))
def load_benchmark(use_task_labels=False, fast_test=True): """ Returns a NC Benchmark from a fake dataset of 10 classes, 5 experiences, 2 classes per experience. """ if fast_test: my_nc_benchmark = get_fast_benchmark(use_task_labels) else: mnist_train = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, transform=Compose([ToTensor()]), ) mnist_test = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, transform=Compose([ToTensor()]), ) my_nc_benchmark = nc_benchmark(mnist_train, mnist_test, 5, task_labels=use_task_labels, seed=1234) return my_nc_benchmark
def test_mt_single_dataset_fixed_order(self): order = [2, 3, 5, 7, 8, 9, 0, 1, 4, 6] mnist_train = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, ) mnist_test = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, ) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, 5, task_labels=True, fixed_class_order=order, class_ids_from_zero_in_each_exp=False, ) all_classes = [] for task_id in range(5): all_classes.extend( my_nc_benchmark.classes_in_experience["train"][task_id]) self.assertEqual(order, all_classes)
def test_sit_single_dataset_fixed_order_subset(self): order = [2, 3, 5, 8, 9, 1, 4, 6] mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True) my_nc_benchmark = nc_benchmark(mnist_train, mnist_test, 4, task_labels=False, fixed_class_order=order) self.assertEqual(4, len(my_nc_benchmark.classes_in_experience['train'])) all_classes = set() for batch_id in range(4): self.assertEqual( 2, len(my_nc_benchmark.classes_in_experience['train'][batch_id])) all_classes.update( my_nc_benchmark.classes_in_experience['train'][batch_id]) self.assertEqual(set(order), all_classes)
def main(args): # Model getter: specify dataset and depth of the network. model = pytorchcv_wrapper.resnet('cifar10', depth=20, pretrained=False) # Or get a more specific model. E.g. wide resnet, with depth 40 and growth # factor 8 for Cifar 10. # model = pytorchcv_wrapper.get_model("wrn40_8_cifar10", pretrained=False) # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") device = "cpu" # --- TRANSFORMATIONS transform = transforms.Compose([ ToTensor(), transforms.Normalize((0.491, 0.482, 0.446), (0.247, 0.243, 0.261)) ]) # --- SCENARIO CREATION cifar_train = CIFAR10(root=expanduser("~") + "/.avalanche/data/cifar10/", train=True, download=True, transform=transform) cifar_test = CIFAR10(root=expanduser("~") + "/.avalanche/data/cifar10/", train=False, download=True, transform=transform) scenario = nc_benchmark( cifar_train, cifar_test, 5, task_labels=False, seed=1234, fixed_class_order=[i for i in range(10)]) # choose some metrics and evaluation method interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), forgetting_metrics(experience=True), loggers=[interactive_logger]) # CREATE THE STRATEGY INSTANCE (Naive, with Replay) cl_strategy = Naive(model, torch.optim.SGD(model.parameters(), lr=0.01), CrossEntropyLoss(), train_mb_size=100, train_epochs=1, eval_mb_size=100, device=device, plugins=[ReplayPlugin(mem_size=1000)], evaluator=eval_plugin ) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience ", experience.current_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def fast_scenario(use_task_labels=False, shuffle=True): """ Copied directly from Avalanche in "tests/unit_tests_utils.py". Not used anywhere atm, but could be used as inspiration for writing quicker tests in Sequoia. """ n_samples_per_class = 100 dataset = make_classification( n_samples=10 * n_samples_per_class, n_classes=10, n_features=6, n_informative=6, n_redundant=0, ) X = torch.from_numpy(dataset[0]).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split( X, y, train_size=0.6, shuffle=True, stratify=y ) train_dataset = TensorDataset(train_X, train_y) test_dataset = TensorDataset(test_X, test_y) my_nc_benchmark = nc_benchmark( train_dataset, test_dataset, 5, task_labels=use_task_labels, shuffle=shuffle ) return my_nc_benchmark
def test_nc_benchmark_transformations_advanced(self): # Regression for #577 ds = CIFAR100(root=expanduser("~") + "/.avalanche/data/cifar100/", train=True, download=True) scenario = nc_benchmark(ds, ds, n_experiences=10, shuffle=True, seed=1234, task_labels=False, train_transform=ToTensor(), eval_transform=None) ds_train_train = scenario.train_stream[0].dataset self.assertIsInstance(ds_train_train[0][0], Tensor) ds_train_eval = scenario.train_stream[0].dataset.eval() self.assertIsInstance(ds_train_eval[0][0], Image) ds_test_eval = scenario.test_stream[0].dataset self.assertIsInstance(ds_test_eval[0][0], Image) ds_test_train = scenario.test_stream[0].dataset.train() self.assertIsInstance(ds_test_train[0][0], Tensor)
def load_ar1_benchmark(self): """ Returns a NC benchmark from a fake dataset of 10 classes, 5 experiences, 2 classes per experience. This toy benchmark is intended """ n_samples_per_class = 5 dataset = make_classification( n_samples=10 * n_samples_per_class, n_classes=9, n_features=224 * 224 * 3, n_informative=6, n_redundant=0, ) X = torch.from_numpy(dataset[0]).reshape(-1, 3, 224, 224).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split( X, y, train_size=0.6, shuffle=True, stratify=y ) train_dataset = TensorDataset(train_X, train_y) test_dataset = TensorDataset(test_X, test_y) my_nc_benchmark = nc_benchmark( train_dataset, test_dataset, 3, task_labels=False ) return my_nc_benchmark
def test_sit_single_dataset_remap_indexes(self): order = [2, 3, 5, 8, 9, 1, 4, 6] mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, 4, task_labels=False, fixed_class_order=order, class_ids_from_zero_from_first_exp=True) self.assertEqual(4, len(my_nc_benchmark.classes_in_experience)) all_classes = [] for batch_id in range(4): self.assertEqual( 2, len(my_nc_benchmark.classes_in_experience[batch_id])) all_classes.extend(my_nc_benchmark.classes_in_experience[batch_id]) self.assertEqual(list(range(8)), all_classes) # Regression test for issue #258 for i, experience in enumerate(my_nc_benchmark.train_stream): unique_dataset_classes = sorted(set(experience.dataset.targets)) expected_dataset_classes = list(range(2 * i, 2 * (i+1))) self.assertListEqual(expected_dataset_classes, unique_dataset_classes) self.assertListEqual( sorted(order[2 * i:2 * (i+1)]), sorted(my_nc_benchmark.original_classes_in_exp[i]))
def test_sit_multi_dataset_merge(self): split_mapping = [0, 1, 2, 3, 4, 0, 1, 2, 3, 4] mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True) train_part1 = make_nc_transformation_subset( mnist_train, None, None, range(5)) train_part2 = make_nc_transformation_subset( mnist_train, None, None, range(5, 10)) train_part2 = AvalancheSubset( train_part2, class_mapping=split_mapping) test_part1 = make_nc_transformation_subset( mnist_test, None, None, range(5)) test_part2 = make_nc_transformation_subset( mnist_test, None, None, range(5, 10)) test_part2 = AvalancheSubset(test_part2, class_mapping=split_mapping) my_nc_benchmark = nc_benchmark( [train_part1, train_part2], [test_part1, test_part2], 5, task_labels=False, shuffle=True, seed=1234) self.assertEqual(5, my_nc_benchmark.n_experiences) self.assertEqual(10, my_nc_benchmark.n_classes) for batch_id in range(5): self.assertEqual( 2, len(my_nc_benchmark.classes_in_experience[batch_id])) all_classes = set() for batch_id in range(5): all_classes.update(my_nc_benchmark.classes_in_experience[batch_id]) self.assertEqual(10, len(all_classes))
def test_sit_single_dataset(self): mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True) my_nc_benchmark = nc_benchmark(mnist_train, mnist_test, 5, task_labels=False, shuffle=True, seed=1234) self.assertEqual(5, my_nc_benchmark.n_experiences) self.assertEqual(10, my_nc_benchmark.n_classes) for batch_id in range(my_nc_benchmark.n_experiences): self.assertEqual( 2, len(my_nc_benchmark.classes_in_experience[batch_id])) all_classes = set() for batch_id in range(5): all_classes.update(my_nc_benchmark.classes_in_experience[batch_id]) self.assertEqual(10, len(all_classes))
def test_mt_single_dataset_without_class_id_remap(self): mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True) my_nc_benchmark = nc_benchmark(mnist_train, mnist_test, 5, task_labels=True, shuffle=True, seed=1234, class_ids_from_zero_in_each_exp=False) self.assertEqual(5, my_nc_benchmark.n_experiences) self.assertEqual(10, my_nc_benchmark.n_classes) for task_id in range(5): self.assertEqual( 2, len(my_nc_benchmark.classes_in_experience['train'][task_id])) all_classes = set() for task_id in range(my_nc_benchmark.n_experiences): all_classes.update( my_nc_benchmark.classes_in_experience['train'][task_id]) self.assertEqual(10, len(all_classes))
def test_sit_single_dataset_fixed_order_subset(self): order = [2, 3, 5, 8, 9, 1, 4, 6] mnist_train = MNIST( root=default_dataset_location("mnist"), train=True, download=True, ) mnist_test = MNIST( root=default_dataset_location("mnist"), train=False, download=True, ) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, 4, task_labels=False, fixed_class_order=order, ) self.assertEqual(4, len(my_nc_benchmark.classes_in_experience["train"])) all_classes = set() for batch_id in range(4): self.assertEqual( 2, len(my_nc_benchmark.classes_in_experience["train"][batch_id]) ) all_classes.update( my_nc_benchmark.classes_in_experience["train"][batch_id] ) self.assertEqual(set(order), all_classes)
def test_sit_single_dataset(self): mnist_train = MNIST( root=default_dataset_location("mnist"), train=True, download=True, ) mnist_test = MNIST( root=default_dataset_location("mnist"), train=False, download=True, ) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, 5, task_labels=False, shuffle=True, seed=1234, ) self.assertEqual(5, my_nc_benchmark.n_experiences) self.assertEqual(10, my_nc_benchmark.n_classes) for batch_id in range(my_nc_benchmark.n_experiences): self.assertEqual( 2, len(my_nc_benchmark.classes_in_experience["train"][batch_id]) ) all_classes = set() for batch_id in range(5): all_classes.update( my_nc_benchmark.classes_in_experience["train"][batch_id] ) self.assertEqual(10, len(all_classes))
def test_sit_single_dataset_fixed_order(self): order = [2, 3, 5, 7, 8, 9, 0, 1, 4, 6] mnist_train = MNIST( root=default_dataset_location("mnist"), train=True, download=True, ) mnist_test = MNIST( root=default_dataset_location("mnist"), train=False, download=True, ) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, 5, task_labels=False, fixed_class_order=order, ) all_classes = [] for batch_id in range(5): all_classes.extend( my_nc_benchmark.classes_in_experience["train"][batch_id] ) self.assertEqual(order, all_classes)
def test_nc_benchmark_transformations_advanced(self): # Regression for #577 ds = CIFAR100( root=default_dataset_location("cifar100"), train=True, download=True, ) benchmark = nc_benchmark( ds, ds, n_experiences=10, shuffle=True, seed=1234, task_labels=False, train_transform=ToTensor(), eval_transform=None, ) ds_train_train = benchmark.train_stream[0].dataset self.assertIsInstance(ds_train_train[0][0], Tensor) ds_train_eval = benchmark.train_stream[0].dataset.eval() self.assertIsInstance(ds_train_eval[0][0], Image) ds_test_eval = benchmark.test_stream[0].dataset self.assertIsInstance(ds_test_eval[0][0], Image) ds_test_train = benchmark.test_stream[0].dataset.train() self.assertIsInstance(ds_test_train[0][0], Tensor)
def get_fast_benchmark(use_task_labels=False, shuffle=True, n_samples_per_class=100): dataset = make_classification( n_samples=10 * n_samples_per_class, n_classes=10, n_features=6, n_informative=6, n_redundant=0, ) X = torch.from_numpy(dataset[0]).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split(X, y, train_size=0.6, shuffle=True, stratify=y) train_dataset = TensorDataset(train_X, train_y) test_dataset = TensorDataset(test_X, test_y) my_nc_benchmark = nc_benchmark( train_dataset, test_dataset, 5, task_labels=use_task_labels, shuffle=shuffle, ) return my_nc_benchmark
def __init__(self, scenario="ni", run_id=0): """Init. :param scenario: The desired CoRE50 scenario. Supports 'nc', 'ni', and 'joint', which is the scenario with a single experience. :param run_id: an integer in [0, 4]. Each run uses a different set of expert models and data splits. """ assert scenario in { "ni", "joint", "nc", }, "`scenario` argument must be one of {'ni', 'joint', 'nc'}." core50_normalization = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) core50_train_transforms = Compose([ RandomHorizontalFlip(p=0.5), RandomCrop(size=128, padding=1), RandomRotation(15), ToTensor(), core50_normalization, ]) core50_eval_transforms = Compose( [CenterCrop(size=128), ToTensor(), core50_normalization]) if scenario == "ni": benchmark = CORe50( scenario="ni", train_transform=core50_train_transforms, eval_transform=core50_eval_transforms, run=run_id, ) elif scenario == "nc": benchmark = CORe50( scenario="nc", train_transform=core50_train_transforms, eval_transform=core50_eval_transforms, run=run_id, ) elif scenario == "joint": core50nc = CORe50(scenario="nc") train_cat = AvalancheConcatDataset( [e.dataset for e in core50nc.train_stream]) test_cat = AvalancheConcatDataset( [e.dataset for e in core50nc.test_stream]) benchmark = nc_benchmark(train_cat, test_cat, n_experiences=1, task_labels=False) else: assert False, "Should never get here." ll = len(benchmark.train_stream) experts = _load_expert_models(f"{scenario}_core50", run_id, ll) super().__init__(benchmark, experts)
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) test_transform = transforms.Compose( [ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # --------- # --- SCENARIO CREATION mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, transform=train_transform) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, transform=test_transform) scenario = nc_benchmark(mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def test_nc_benchmark_transformations_basic(self): # Regression for #577 ds = CIFAR100(root=expanduser("~") + "/.avalanche/data/cifar100/", train=True, download=True) ds = AvalancheDataset(ds, transform=ToTensor()) benchmark = nc_benchmark( ds, ds, n_experiences=10, shuffle=True, seed=1234, task_labels=False ) exp_0_dataset = benchmark.train_stream[0].dataset self.assertIsInstance(exp_0_dataset[0][0], Tensor)
def test_sit_multi_dataset_one_batch_per_set(self): split_mapping = [0, 1, 2, 0, 1, 2, 3, 4, 5, 6] mnist_train = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, ) mnist_test = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, ) train_part1 = make_nc_transformation_subset(mnist_train, None, None, range(3)) train_part2 = make_nc_transformation_subset(mnist_train, None, None, range(3, 10)) train_part2 = AvalancheSubset(train_part2, class_mapping=split_mapping) test_part1 = make_nc_transformation_subset(mnist_test, None, None, range(3)) test_part2 = make_nc_transformation_subset(mnist_test, None, None, range(3, 10)) test_part2 = AvalancheSubset(test_part2, class_mapping=split_mapping) my_nc_benchmark = nc_benchmark( [train_part1, train_part2], [test_part1, test_part2], 2, task_labels=False, shuffle=True, seed=1234, one_dataset_per_exp=True, ) self.assertEqual(2, my_nc_benchmark.n_experiences) self.assertEqual(10, my_nc_benchmark.n_classes) all_classes = set() for batch_id in range(2): all_classes.update( my_nc_benchmark.classes_in_experience["train"][batch_id]) self.assertEqual(10, len(all_classes)) self.assertTrue( (my_nc_benchmark.classes_in_experience["train"][0] == {0, 1, 2} and my_nc_benchmark.classes_in_experience["train"][1] == set( range(3, 10))) or (my_nc_benchmark.classes_in_experience["train"][0] == set( range(3, 10)) and my_nc_benchmark.classes_in_experience["train"][1] == {0, 1, 2}))
def test_nc_sit_slicing(self): mnist_train = MNIST( root=default_dataset_location("mnist"), train=True, download=True, ) mnist_test = MNIST( root=default_dataset_location("mnist"), train=False, download=True, ) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, 5, task_labels=False, shuffle=True, seed=1234, ) experience: NCExperience for batch_id, experience in enumerate(my_nc_benchmark.train_stream): self.assertEqual(batch_id, experience.current_experience) self.assertIsInstance(experience, NCExperience) for batch_id, experience in enumerate(my_nc_benchmark.test_stream): self.assertEqual(batch_id, experience.current_experience) self.assertIsInstance(experience, NCExperience) iterable_slice = [3, 4, 1] sliced_stream = my_nc_benchmark.train_stream[iterable_slice] self.assertIsInstance(sliced_stream, ClassificationStream) self.assertEqual(len(iterable_slice), len(sliced_stream)) self.assertEqual("train", sliced_stream.name) for batch_id, experience in enumerate(sliced_stream): self.assertEqual( iterable_slice[batch_id], experience.current_experience ) self.assertIsInstance(experience, NCExperience) sliced_stream = my_nc_benchmark.test_stream[iterable_slice] self.assertIsInstance(sliced_stream, ClassificationStream) self.assertEqual(len(iterable_slice), len(sliced_stream)) self.assertEqual("test", sliced_stream.name) for batch_id, experience in enumerate(sliced_stream): self.assertEqual( iterable_slice[batch_id], experience.current_experience ) self.assertIsInstance(experience, NCExperience)
def test_sit_single_dataset_reproducibility_data(self): mnist_train = MNIST( root=default_dataset_location("mnist"), train=True, download=True, ) mnist_test = MNIST( root=default_dataset_location("mnist"), train=False, download=True, ) nc_benchmark_ref = nc_benchmark( mnist_train, mnist_test, 5, task_labels=False, shuffle=True, seed=5678, ) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, -1, task_labels=False, reproducibility_data=nc_benchmark_ref.get_reproducibility_data(), ) self.assertEqual( nc_benchmark_ref.train_exps_patterns_assignment, my_nc_benchmark.train_exps_patterns_assignment, ) self.assertEqual( nc_benchmark_ref.test_exps_patterns_assignment, my_nc_benchmark.test_exps_patterns_assignment, )
def test_nc_sit_slicing(self): mnist_train = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, ) mnist_test = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, ) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, 5, task_labels=False, shuffle=True, seed=1234, ) experience: NCExperience for batch_id, experience in enumerate(my_nc_benchmark.train_stream): self.assertEqual(batch_id, experience.current_experience) self.assertIsInstance(experience, NCExperience) for batch_id, experience in enumerate(my_nc_benchmark.test_stream): self.assertEqual(batch_id, experience.current_experience) self.assertIsInstance(experience, NCExperience) iterable_slice = [3, 4, 1] sliced_stream = my_nc_benchmark.train_stream[iterable_slice] self.assertIsInstance(sliced_stream, GenericScenarioStream) self.assertEqual(len(iterable_slice), len(sliced_stream)) self.assertEqual("train", sliced_stream.name) for batch_id, experience in enumerate(sliced_stream): self.assertEqual(iterable_slice[batch_id], experience.current_experience) self.assertIsInstance(experience, NCExperience) sliced_stream = my_nc_benchmark.test_stream[iterable_slice] self.assertIsInstance(sliced_stream, GenericScenarioStream) self.assertEqual(len(iterable_slice), len(sliced_stream)) self.assertEqual("test", sliced_stream.name) for batch_id, experience in enumerate(sliced_stream): self.assertEqual(iterable_slice[batch_id], experience.current_experience) self.assertIsInstance(experience, NCExperience)
def test_sit_single_dataset_fixed_subset_no_remap_idx(self): order = [2, 5, 7, 8, 9, 0, 1, 4] mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True) my_nc_benchmark = nc_benchmark( mnist_train, mnist_test, 2, task_labels=True, fixed_class_order=order, class_ids_from_zero_in_each_exp=False) self.assertEqual(2, len(my_nc_benchmark.classes_in_experience)) all_classes = set() for task_id in range(2): self.assertEqual( 4, len(my_nc_benchmark.classes_in_experience[task_id]) ) self.assertEqual(set(order[task_id*4:(task_id+1)*4]), my_nc_benchmark.original_classes_in_exp[task_id]) all_classes.update(my_nc_benchmark.classes_in_experience[task_id]) self.assertEqual(set(order), all_classes)