def test_SplitCifar100_benchmark_download_once(self): global CIFAR100_DOWNLOADS, CIFAR10_DOWNLOADS CIFAR100_DOWNLOADS = 0 CIFAR10_DOWNLOADS = 0 benchmark = SplitCIFAR100(5) self.assertEqual(5, len(benchmark.train_stream)) self.assertEqual(5, len(benchmark.test_stream)) self.assertEqual(1, CIFAR100_DOWNLOADS) self.assertEqual(0, CIFAR10_DOWNLOADS)
def test_SplitCifar100_scenario_download_once(self): global CIFAR100_DOWNLOADS, CIFAR10_DOWNLOADS CIFAR100_DOWNLOADS = 0 CIFAR10_DOWNLOADS = 0 scenario = SplitCIFAR100(5) self.assertEqual(5, len(scenario.train_stream)) self.assertEqual(5, len(scenario.test_stream)) self.assertEqual(1, CIFAR100_DOWNLOADS) self.assertEqual(0, CIFAR10_DOWNLOADS)
def evaluate_on_cifar_100( *, method_name: str, plugins: List[StrategyPlugin], tb_dir: str = str(TB_DIR), seed: int = 42, verbose: bool = False, train_epochs: int = 70, n_classes_per_batch: int = 10, start_lr: float = 2.0, lr_milestones: List[int] = None, lr_gamma: float = 0.2, ): assert not N_CLASSES % n_classes_per_batch, "n_classes should be a multiple of n_classes_per_batch" scenario = SplitCIFAR100(n_experiences=N_CLASSES // n_classes_per_batch) model = ResNet32(n_classes=N_CLASSES) tb_logger = TensorboardLogger(tb_dir + f"/cifar100_{n_classes_per_batch}/{method_name}/{seed}_{create_time_id()}") loggers = [tb_logger] if verbose: loggers.append(InteractiveLogger()) strategy = Naive( model=model, optimizer=SGD(model.parameters(), lr=2.0, weight_decay=0.00001), criterion=CrossEntropyLoss(), train_epochs=train_epochs, train_mb_size=128, device=device, plugins=plugins + [LRSchedulerPlugin(start_lr=start_lr, milestones=lr_milestones, gamma=lr_gamma)], evaluator=EvaluationPlugin( [ NormalizedStreamAccuracy(), NormalizedExperienceAccuracy(), ExperienceMeanRepresentationShift(MeanL2RepresentationShift()), ExperienceMeanRepresentationShift(MeanCosineRepresentationShift()), ], StreamConfusionMatrix( num_classes=N_CLASSES, image_creator=SortedCMImageCreator(scenario.classes_order), ), loggers=loggers, ), ) for i, train_task in enumerate(scenario.train_stream, 1): strategy.train(train_task, num_workers=0) strategy.eval(scenario.test_stream[:i]) tb_logger.writer.flush()
def test_SplitCifar100_benchmark(self): benchmark = SplitCIFAR100(5) self.assertEqual(5, len(benchmark.train_stream)) self.assertEqual(5, len(benchmark.test_stream)) train_sz = 0 for experience in benchmark.train_stream: self.assertIsInstance(experience, Experience) train_sz += len(experience.dataset) # Regression test for 575 load_experience_train_eval(experience) self.assertEqual(50000, train_sz) test_sz = 0 for experience in benchmark.test_stream: self.assertIsInstance(experience, Experience) test_sz += len(experience.dataset) # Regression test for 575 load_experience_train_eval(experience) self.assertEqual(10000, test_sz)