def test_naive(self): # SIT scenario model, optimizer, criterion, my_nc_benchmark = self.init_sit() strategy = Naive( model, optimizer, criterion, train_mb_size=64, device=self.device, eval_mb_size=50, train_epochs=2, ) self.run_strategy(my_nc_benchmark, strategy) # MT scenario strategy = Naive( model, optimizer, criterion, train_mb_size=64, device=self.device, eval_mb_size=50, train_epochs=2, ) benchmark = self.load_benchmark(use_task_labels=True) self.run_strategy(benchmark, strategy)
def assert_balancing(self, policy): benchmark = get_fast_benchmark(use_task_labels=True) replay = ReplayPlugin(mem_size=100, storage_policy=policy) model = SimpleMLP(num_classes=benchmark.n_classes) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001), CrossEntropyLoss(), train_mb_size=100, train_epochs=0, eval_mb_size=100, plugins=[replay], evaluator=None, ) for exp in benchmark.train_stream: cl_strategy.train(exp) ext_mem = policy.buffer_groups ext_mem_data = policy.buffer_datasets print(list(ext_mem.keys()), [len(el) for el in ext_mem_data]) # buffer size should equal self.mem_size if data is large enough len_tot = sum([len(el) for el in ext_mem_data]) assert len_tot == policy.max_size
def test_plugins_compatibility_checks(self): model = SimpleMLP(input_size=6, hidden_size=10) benchmark = get_fast_benchmark() optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() evalp = EvaluationPlugin( loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), loggers=[InteractiveLogger()], strict_checks=None, ) strategy = Naive( model, optimizer, criterion, train_epochs=2, eval_every=-1, evaluator=evalp, plugins=[ EarlyStoppingPlugin(patience=10, val_stream_name="train") ], ) strategy.train(benchmark.train_stream[0])
def test_incremental_classifier_weight_update(self): model = IncrementalClassifier(in_features=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() benchmark = self.benchmark strategy = Naive( model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device="cpu", ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] # train on first task w_old = model.classifier.weight.clone() b_old = model.classifier.bias.clone() # adaptation. Increase number of classes dataset = benchmark.train_stream[4].dataset model.adaptation(dataset) w_new = model.classifier.weight.clone() b_new = model.classifier.bias.clone() # old weights should be copied correctly. assert torch.equal(w_old, w_new[:w_old.shape[0]]) assert torch.equal(b_old, b_new[:w_old.shape[0]]) # shape should be correct. assert w_new.shape[0] == max(dataset.targets) + 1 assert b_new.shape[0] == max(dataset.targets) + 1
def main(args): # Config device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # model model = SimpleMLP(input_size=32 * 32 * 3, num_classes=10) # CL Benchmark Creation scenario = SplitCIFAR10(n_experiences=5, return_task_id=True) train_stream = scenario.train_stream test_stream = scenario.test_stream # Prepare for training & testing optimizer = Adam(model.parameters(), lr=0.01) criterion = CrossEntropyLoss() # Choose a CL strategy strategy = Naive( model=model, optimizer=optimizer, criterion=criterion, train_mb_size=128, train_epochs=3, eval_mb_size=128, device=device, ) # train and test loop for train_task in train_stream: strategy.train(train_task, num_workers=0) strategy.eval(test_stream)
def _test_replay_balanced_memory(self, storage_policy, mem_size): benchmark = get_fast_benchmark(use_task_labels=True) model = SimpleMLP(input_size=6, hidden_size=10) replayPlugin = ReplayPlugin(mem_size=mem_size, storage_policy=storage_policy) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001), CrossEntropyLoss(), train_mb_size=32, train_epochs=1, eval_mb_size=100, plugins=[replayPlugin], ) n_seen_data = 0 for step in benchmark.train_stream: n_seen_data += len(step.dataset) mem_fill = min(mem_size, n_seen_data) cl_strategy.train(step) lengths = [] for d in replayPlugin.storage_policy.buffer_datasets: lengths.append(len(d)) self.assertEqual(sum(lengths), mem_fill) # Always fully filled
def test_multihead_head_creation(self): # Check if the optimizer is updated correctly # when heads are created and updated. model = MTSimpleMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() benchmark = get_fast_benchmark(use_task_labels=True, shuffle=False) strategy = Naive( model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device="cpu", ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] print( "Current Classes: ", benchmark.train_stream[4].classes_in_this_experience, ) print( "Current Classes: ", benchmark.train_stream[0].classes_in_this_experience, ) # head creation strategy.train(benchmark.train_stream[0]) w_ptr = model.classifier.classifiers["0"].classifier.weight.data_ptr() b_ptr = model.classifier.classifiers["0"].classifier.bias.data_ptr() opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group["params"] ] assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # head update strategy.train(benchmark.train_stream[4]) w_ptr_t0 = model.classifier.classifiers[ "0"].classifier.weight.data_ptr() b_ptr_t0 = model.classifier.classifiers["0"].classifier.bias.data_ptr() w_ptr_new = model.classifier.classifiers[ "4"].classifier.weight.data_ptr() b_ptr_new = model.classifier.classifiers["4"].classifier.bias.data_ptr( ) opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group["params"] ] assert w_ptr not in opt_params_ptrs # head0 has been updated assert b_ptr not in opt_params_ptrs # head0 has been updated assert w_ptr_t0 in opt_params_ptrs assert b_ptr_t0 in opt_params_ptrs assert w_ptr_new in opt_params_ptrs assert b_ptr_new in opt_params_ptrs
def test_rwalk(self): # SIT scenario model, optimizer, criterion, my_nc_benchmark = self.init_sit() strategy = Naive( model, optimizer, criterion, train_mb_size=10, eval_mb_size=50, train_epochs=2, plugins=[ RWalkPlugin( ewc_lambda=0.1, ewc_alpha=0.9, delta_t=10, ), ], ) self.run_strategy(my_nc_benchmark, strategy) # MT scenario strategy = Naive( model, optimizer, criterion, train_mb_size=10, eval_mb_size=50, train_epochs=2, plugins=[ RWalkPlugin( ewc_lambda=0.1, ewc_alpha=0.9, delta_t=10, ), ], ) benchmark = self.load_benchmark(use_task_labels=True) self.run_strategy(benchmark, strategy)
def main(args): # Device config device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) # model model = SimpleMLP(num_classes=10) # Here we show all the MNIST variation we offer in the "classic" benchmarks if args.mnist_type == "permuted": scenario = PermutedMNIST(n_experiences=5, seed=1) elif args.mnist_type == "rotated": scenario = RotatedMNIST( n_experiences=5, rotations_list=[30, 60, 90, 120, 150], seed=1 ) else: scenario = SplitMNIST(n_experiences=5, seed=1) # Than we can extract the parallel train and test streams train_stream = scenario.train_stream test_stream = scenario.test_stream # Prepare for training & testing optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9) criterion = CrossEntropyLoss() # Continual learning strategy with default logger cl_strategy = Naive( model, optimizer, criterion, train_mb_size=32, train_epochs=100, eval_mb_size=32, device=device, eval_every=1, plugins=[EarlyStoppingPlugin(args.patience, "test_stream")], ) # train and test loop results = [] for train_task, test_task in zip(train_stream, test_stream): print("Current Classes: ", train_task.classes_in_this_experience) cl_strategy.train(train_task, eval_streams=[test_task]) results.append(cl_strategy.eval(test_stream))
def test_optimizer_update(self): model = SimpleMLP() optimizer = SGD(model.parameters(), lr=1e-3) strategy = Naive(model, optimizer, None) # check add_param_group p = torch.nn.Parameter(torch.zeros(10, 10)) add_new_params_to_optimizer(optimizer, p) assert self._is_param_in_optimizer(p, strategy.optimizer) # check new_param is in optimizer # check old_param is NOT in optimizer p_new = torch.nn.Parameter(torch.zeros(10, 10)) update_optimizer(optimizer, [p], [p_new]) assert self._is_param_in_optimizer(p_new, strategy.optimizer) assert not self._is_param_in_optimizer(p, strategy.optimizer)
def main(cuda: int): # --- CONFIG device = torch.device( f"cuda:{cuda}" if torch.cuda.is_available() else "cpu") # --- SCENARIO CREATION scenario = SplitCIFAR10(n_experiences=2, seed=42) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes, input_size=196608 // 64) # choose some metrics and evaluation method eval_plugin = EvaluationPlugin( accuracy_metrics(stream=True, experience=True), images_samples_metrics( on_train=True, on_eval=True, n_cols=10, n_rows=10, ), labels_repartition_metrics( # image_creator=repartition_bar_chart_image_creator, on_train=True, on_eval=True, ), loggers=[ TensorboardLogger(f"tb_data/{datetime.now()}"), InteractiveLogger(), ], ) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, Adam(model.parameters()), train_mb_size=128, train_epochs=1, eval_mb_size=128, device=device, plugins=[ReplayPlugin(mem_size=1_000)], evaluator=eval_plugin, ) # TRAINING LOOP for i, experience in enumerate(scenario.train_stream, 1): cl_strategy.train(experience) cl_strategy.eval(scenario.test_stream[:i])
def test_dataload_reinit(self): benchmark = get_fast_benchmark() model = SimpleMLP(input_size=6, hidden_size=10) replayPlugin = ReplayPlugin(mem_size=5) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001), CrossEntropyLoss(), train_mb_size=16, train_epochs=1, eval_mb_size=16, plugins=[replayPlugin], ) for step in benchmark.train_stream[:2]: cl_strategy.train(step)
def test_no_errors(self): eval_plugin = EvaluationPlugin( accuracy_metrics(stream=True), loggers=None, benchmark=self.benchmark, strict_checks=True, ) strategy = Naive( self.model, self.optimizer, self.criterion, train_epochs=2, eval_every=0, evaluator=eval_plugin, ) for exp in self.benchmark.train_stream: strategy.train(exp, eval_streams=[self.benchmark.test_stream]) strategy.eval(self.benchmark.test_stream)
def test_forward_hooks(self): model = SimpleMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() strategy = Naive( model, optimizer, criterion, train_epochs=2, eval_every=0 ) was_hook_called = False def hook(a, b, c): nonlocal was_hook_called was_hook_called = True model.register_forward_hook(hook) mb_x = torch.randn(32, 6, device=strategy.device) strategy.mbatch = mb_x, None, None strategy.forward() assert was_hook_called
def test_raise_warning(self): eval_plugin = EvaluationPlugin( accuracy_metrics(stream=True), loggers=None, benchmark=self.benchmark, strict_checks=False, ) strategy = Naive( self.model, self.optimizer, self.criterion, train_epochs=2, eval_every=-1, evaluator=eval_plugin, ) for exp in self.benchmark.train_stream: strategy.train(exp) strategy.eval(self.benchmark.test_stream) with self.assertWarns(UserWarning): strategy.eval(self.benchmark.test_stream[:2])
def _test_integration(self, module, clf_name, plugins=[]): module = as_multitask(module, clf_name) module = module.to(self.device) optimizer = SGD(module.parameters(), lr=0.05, momentum=0.9, weight_decay=0.0002) strategy = Naive( module, optimizer, train_mb_size=32, eval_mb_size=32, device=self.device, plugins=plugins, ) for t, experience in enumerate(self.benchmark.train_stream): strategy.train(experience) strategy.eval(self.benchmark.test_stream[:t + 1])
def test_dataload_batch_balancing(self): benchmark = get_fast_benchmark() batch_size = 32 replayPlugin = ReplayPlugin(mem_size=20) model = SimpleMLP(input_size=6, hidden_size=10) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001), CrossEntropyLoss(), train_mb_size=batch_size, train_epochs=1, eval_mb_size=100, plugins=[replayPlugin], ) for step in benchmark.train_stream: adapted_dataset = step.dataset if len(replayPlugin.storage_policy.buffer) > 0: dataloader = ReplayDataLoader( adapted_dataset, replayPlugin.storage_policy.buffer, oversample_small_tasks=True, num_workers=0, batch_size=batch_size, shuffle=True, ) else: dataloader = TaskBalancedDataLoader(adapted_dataset) for mini_batch in dataloader: mb_task_labels = mini_batch[-1] lengths = [] for task_id in adapted_dataset.task_set: len_task = (mb_task_labels == task_id).sum() lengths.append(len_task) if sum(lengths) == batch_size: difference = max(lengths) - min(lengths) self.assertLessEqual(difference, 1) self.assertLessEqual(sum(lengths), batch_size) cl_strategy.train(step)
def main(args): # Config device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # Model model = SimpleCNN(num_classes=5) # CL Benchmark Creation scenario = EndlessCLSim( scenario=args.scenario, # "Classes", "Illumination", "Weather" sequence_order=None, task_order=None, semseg=args.semseg, dataset_root=args.dataset_root, ) train_stream = scenario.train_stream test_stream = scenario.test_stream # Prepare for training & testing optimizer = Adam(model.parameters(), lr=0.001) criterion = CrossEntropyLoss() # Choose a CL strategy strategy = Naive( model=model, optimizer=optimizer, criterion=criterion, train_mb_size=64, train_epochs=3, eval_mb_size=128, device=device, ) # Train and test loop for train_task in train_stream: strategy.train(train_task, num_worker=0) strategy.eval(test_stream) return
def test_multihead_head_selection(self): # Check if the optimizer is updated correctly # when heads are created and updated. model = MultiHeadClassifier(in_features=6) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() benchmark = get_fast_benchmark(use_task_labels=True, shuffle=False) strategy = Naive( model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device="cpu", ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] # initialize head strategy.train(benchmark.train_stream[0]) strategy.train(benchmark.train_stream[4]) # create models with fixed head model_t0 = model.classifiers["0"] model_t4 = model.classifiers["4"] # check head task0 for x, y, t in DataLoader(benchmark.train_stream[0].dataset): y_mh = model(x, t) y_t = model_t0(x) assert ((y_mh - y_t)**2).sum() < 1.0e-7 break # check head task4 for x, y, t in DataLoader(benchmark.train_stream[4].dataset): y_mh = model(x, t) y_t = model_t4(x) assert ((y_mh - y_t)**2).sum() < 1.0e-7 break
def test_callback_reachability(self): # Check that all the callbacks are called during # training and test loops. model = _PlainMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() benchmark = PluginTests.create_benchmark() plug = MockPlugin() strategy = Naive( model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device="cpu", plugins=[plug], ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] strategy.train(benchmark.train_stream[0], num_workers=0) strategy.eval([benchmark.test_stream[0]], num_workers=0) assert all(plug.activated)
def test_incremental_classifier(self): model = SimpleMLP(input_size=6, hidden_size=10) model.classifier = IncrementalClassifier(in_features=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() benchmark = self.benchmark strategy = Naive( model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device="cpu", ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] print( "Current Classes: ", benchmark.train_stream[0].classes_in_this_experience, ) print( "Current Classes: ", benchmark.train_stream[4].classes_in_this_experience, ) # train on first task strategy.train(benchmark.train_stream[0]) w_ptr = model.classifier.classifier.weight.data_ptr() b_ptr = model.classifier.classifier.bias.data_ptr() opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group["params"] ] # classifier params should be optimized assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # train again on the same task. strategy.train(benchmark.train_stream[0]) # parameters should not change. assert w_ptr == model.classifier.classifier.weight.data_ptr() assert b_ptr == model.classifier.classifier.bias.data_ptr() # the same classifier params should still be optimized assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # update classifier with new classes. old_w_ptr, old_b_ptr = w_ptr, b_ptr strategy.train(benchmark.train_stream[4]) opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group["params"] ] new_w_ptr = model.classifier.classifier.weight.data_ptr() new_b_ptr = model.classifier.classifier.bias.data_ptr() # weights should change. assert old_w_ptr != new_w_ptr assert old_b_ptr != new_b_ptr # Old params should not be optimized. New params should be optimized. assert old_w_ptr not in opt_params_ptrs assert old_b_ptr not in opt_params_ptrs assert new_w_ptr in opt_params_ptrs assert new_b_ptr in opt_params_ptrs
def main(args): # Model getter: specify dataset and depth of the network. model = pytorchcv_wrapper.resnet("cifar10", depth=20, pretrained=False) # Or get a more specific model. E.g. wide resnet, with depth 40 and growth # factor 8 for Cifar 10. # model = pytorchcv_wrapper.get_model("wrn40_8_cifar10", pretrained=False) # --- CONFIG device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) device = "cpu" # --- TRANSFORMATIONS transform = transforms.Compose( [ ToTensor(), transforms.Normalize((0.491, 0.482, 0.446), (0.247, 0.243, 0.261)), ] ) # --- SCENARIO CREATION cifar_train = CIFAR10( root=expanduser("~") + "/.avalanche/data/cifar10/", train=True, download=True, transform=transform, ) cifar_test = CIFAR10( root=expanduser("~") + "/.avalanche/data/cifar10/", train=False, download=True, transform=transform, ) scenario = nc_benchmark( cifar_train, cifar_test, 5, task_labels=False, seed=1234, fixed_class_order=[i for i in range(10)], ) # choose some metrics and evaluation method interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, experience=True, stream=True ), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), forgetting_metrics(experience=True), loggers=[interactive_logger], ) # CREATE THE STRATEGY INSTANCE (Naive, with Replay) cl_strategy = Naive( model, torch.optim.SGD(model.parameters(), lr=0.01), CrossEntropyLoss(), train_mb_size=100, train_epochs=1, eval_mb_size=100, device=device, plugins=[ReplayPlugin(mem_size=1000)], evaluator=eval_plugin, ) # TRAINING LOOP print("Starting experiment...") results = [] for experience in scenario.train_stream: print("Start of experience ", experience.current_experience) cl_strategy.train(experience) print("Training completed") print("Computing accuracy on the whole test set") results.append(cl_strategy.eval(scenario.test_stream))
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )), ]) test_transform = transforms.Compose( [ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # --------- # --- SCENARIO CREATION mnist_train = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, transform=train_transform, ) mnist_test = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, transform=test_transform, ) scenario = nc_benchmark(mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) interactive_logger = InteractiveLogger() wandb_logger = WandBLogger(project_name=args.project, run_name=args.run, config=vars(args)) eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), loss_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), forgetting_metrics(experience=True, stream=True), confusion_matrix_metrics(stream=True, wandb=True, class_names=[str(i) for i in range(10)]), cpu_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, experience=True, stream=True), ram_usage_metrics(every=0.5, minibatch=True, epoch=True, experience=True, stream=True), gpu_usage_metrics( args.cuda, every=0.5, minibatch=True, epoch=True, experience=True, stream=True, ), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[interactive_logger, wandb_logger], ) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device, evaluator=eval_plugin, ) # TRAINING LOOP print("Starting experiment...") results = [] for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) cl_strategy.train(experience) print("Training completed") print("Computing accuracy on the whole test set") results.append(cl_strategy.eval(scenario.test_stream))
def main(args): # Device config device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # Intialize the model, stream and training strategy model = SimpleCNN(num_classes=10) if args.stream != "s_long": model_init = deepcopy(model) scenario = CTrL(stream_name=args.stream, save_to_disk=args.save, path=args.path, seed=10) train_stream = scenario.train_stream test_stream = scenario.test_stream val_stream = scenario.val_stream optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9) criterion = CrossEntropyLoss() logger = EvaluationPlugin( accuracy_metrics(minibatch=False, epoch=False, experience=True, stream=True), loggers=[InteractiveLogger()], ) cl_strategy = Naive( model, optimizer, criterion, train_mb_size=32, device=device, train_epochs=args.max_epochs, eval_mb_size=128, evaluator=logger, plugins=[EarlyStoppingPlugin(50, "val_stream")], eval_every=5, ) # train and test loop for train_task, val_task in zip(train_stream, val_stream): cl_strategy.train(train_task, eval_streams=[val_task]) cl_strategy.eval(test_stream) transfer_mat = [] for tid in range(len(train_stream)): transfer_mat.append( logger.all_metric_results["Top1_Acc_Exp/eval_phase/test_stream/" f"Task00{tid}/Exp00{tid}"][1]) if args.stream == "s_long": res = logger.last_metric_results["Top1_Acc_Stream/eval_phase/" "test_stream"] print(f"Average accuracy on S_long : {res}") else: optimizer = SGD(model_init.parameters(), lr=0.001, momentum=0.9) cl_strategy = Naive( model_init, optimizer, criterion, train_mb_size=32, device=device, train_epochs=args.max_epochs, eval_mb_size=128, plugins=[EarlyStoppingPlugin(50, "val_stream")], eval_every=5, ) cl_strategy.train(train_stream[-1]) res = cl_strategy.eval([test_stream[-1]]) acc_last_stream = transfer_mat[-1][-1] acc_last_only = res["Top1_Acc_Exp/eval_phase/test_stream/" "Task005/Exp-01"] transfer_value = acc_last_stream - acc_last_only print(f"Accuracy on probe task after training on the whole " f"stream: {acc_last_stream}") print(f"Accuracy on probe task after trained " f"independently: {acc_last_only}") print(f"T({args.stream})={transfer_value}")
def main(args): model = SimpleMLP(hidden_size=args.hs) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) criterion = torch.nn.CrossEntropyLoss() # check if selected GPU is available or use CPU assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0." device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") print(f"Using device: {device}") # create scenario if args.scenario == "pmnist": scenario = PermutedMNIST(n_experiences=args.permutations) elif args.scenario == "smnist": mnist_train = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, transform=ToTensor(), ) mnist_test = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, transform=ToTensor(), ) scenario = nc_benchmark(mnist_train, mnist_test, 5, task_labels=False, seed=1234) else: raise ValueError("Wrong scenario name. Allowed pmnist, smnist.") # choose some metrics and evaluation method interactive_logger = InteractiveLogger() tensorboard_logger = TensorboardLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), forgetting_metrics(experience=True, stream=True), bwt_metrics(experience=True, stream=True), loggers=[interactive_logger, tensorboard_logger], ) # create strategy strategy = Naive( model, optimizer, criterion, train_epochs=args.epochs, device=device, train_mb_size=args.minibatch_size, evaluator=eval_plugin, plugins=[ RWalkPlugin( ewc_lambda=args.ewc_lambda, ewc_alpha=args.ewc_alpha, delta_t=args.delta_t, ) ], ) # train on the selected scenario with the chosen strategy print("Starting experiment...") results = [] for experience in scenario.train_stream: print("Start training on experience ", experience.current_experience) strategy.train(experience) print("End training on experience", experience.current_experience) print("Computing accuracy on the test set") results.append(strategy.eval(scenario.test_stream[:]))
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )), ]) test_transform = transforms.Compose( [ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # --------- # --- SCENARIO CREATION mnist_train = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, transform=train_transform, ) mnist_test = MNIST( root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, transform=test_transform, ) scenario = nc_benchmark(mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) eval_plugin = EvaluationPlugin( accuracy_metrics(epoch=True, experience=True, stream=True), loss_metrics(epoch=True, experience=True, stream=True), # save image should be False to appropriately view # results in Interactive Logger. # a tensor will be printed confusion_matrix_metrics(save_image=False, normalize="all", stream=True), loggers=InteractiveLogger(), ) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device, evaluator=eval_plugin, plugins=[ReplayPlugin(5000)], ) # TRAINING LOOP print("Starting experiment...") results = [] for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) cl_strategy.train(experience) print("Training completed") print("Computing accuracy on the whole test set") results.append(cl_strategy.eval(scenario.test_stream))
def main(args): """ Last Avalanche version reference performance (online = 1 epoch): Class-incremental (online): Top1_Acc_Stream/eval_phase/test_stream = 0.9421 Data-incremental (online: Top1_Acc_Stream/eval_phase/test_stream = 0.9309 These are reference results for a single run. """ # --- DEFAULT PARAMS ONLINE DATA INCREMENTAL LEARNING nb_tasks = 5 # Can still design the data stream based on tasks batch_size = 10 # Learning agent only has small amount of data available epochs = 1 # How many times to process each mini-batch return_task_id = False # Data incremental (task-agnostic/task-free) # --- CONFIG device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) # --------- # --- SCENARIO CREATION n_classes = 10 task_scenario = SplitMNIST( nb_tasks, return_task_id=return_task_id, fixed_class_order=[i for i in range(n_classes)], ) # Make data incremental (one batch = one experience) scenario = data_incremental_benchmark( task_scenario, experience_size=batch_size ) print(f"{scenario.n_experiences} batches in online data incremental setup.") # 6002 batches for SplitMNIST with batch size 10 # --------- # MODEL CREATION model = SimpleMLP( num_classes=args.featsize, hidden_size=400, hidden_layers=2, drop_rate=0 ) # choose some metrics and evaluation method logger = TextLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(experience=True, stream=True), loss_metrics(experience=False, stream=True), StreamForgetting(), loggers=[logger], benchmark=scenario, ) # CoPE PLUGIN cope = CoPEPlugin( mem_size=2000, alpha=0.99, p_size=args.featsize, n_classes=n_classes ) # CREATE THE STRATEGY INSTANCE (NAIVE) WITH CoPE PLUGIN cl_strategy = Naive( model, torch.optim.SGD(model.parameters(), lr=0.01), cope.ppp_loss, # CoPE PPP-Loss train_mb_size=batch_size, train_epochs=epochs, eval_mb_size=100, device=device, plugins=[cope], evaluator=eval_plugin, ) # TRAINING LOOP print("Starting experiment...") results = [] cl_strategy.train(scenario.train_stream) print("Computing accuracy on the whole test set") results.append(cl_strategy.eval(scenario.test_stream))
def main(args): # --- CONFIG device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) # --------- tr_ds = [ AvalancheTensorDataset( torch.randn(10, 3), torch.randint(0, 3, (10,)).tolist(), task_labels=torch.randint(0, 5, (10,)).tolist(), ) for _ in range(3) ] ts_ds = [ AvalancheTensorDataset( torch.randn(10, 3), torch.randint(0, 3, (10,)).tolist(), task_labels=torch.randint(0, 5, (10,)).tolist(), ) for _ in range(3) ] scenario = create_multi_dataset_generic_benchmark( train_datasets=tr_ds, test_datasets=ts_ds ) # --------- # MODEL CREATION model = SimpleMLP(num_classes=3, input_size=3) # DEFINE THE EVALUATION PLUGIN AND LOGGER # The evaluation plugin manages the metrics computation. # It takes as argument a list of metrics and a list of loggers. # The evaluation plugin calls the loggers to serialize the metrics # and save them in persistent memory or print them in the standard output. # log to text file text_logger = TextLogger(open("log.txt", "a")) # print to stdout interactive_logger = InteractiveLogger() csv_logger = CSVLogger() eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), loss_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), forgetting_metrics(experience=True, stream=True), bwt_metrics(experience=True, stream=True), cpu_usage_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), timing_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), ram_usage_metrics( every=0.5, minibatch=True, epoch=True, experience=True, stream=True ), gpu_usage_metrics( args.cuda, every=0.5, minibatch=True, epoch=True, experience=True, stream=True, ), disk_usage_metrics( minibatch=True, epoch=True, experience=True, stream=True ), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[interactive_logger, text_logger, csv_logger], collect_all=True, ) # collect all metrics (set to True by default) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=500, train_epochs=1, eval_mb_size=100, device=device, evaluator=eval_plugin, eval_every=1, ) # TRAINING LOOP print("Starting experiment...") results = [] for i, experience in enumerate(scenario.train_stream): print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) # train returns a dictionary containing last recorded value # for each metric. res = cl_strategy.train(experience, eval_streams=[scenario.test_stream]) print("Training completed") print("Computing accuracy on the whole test set") # test returns a dictionary with the last metric collected during # evaluation on that stream results.append(cl_strategy.eval(scenario.test_stream)) print(f"Test metrics:\n{results}") # Dict with all the metric curves, # only available when `collect_all` is True. # Each entry is a (x, metric value) tuple. # You can use this dictionary to manipulate the # metrics without avalanche. all_metrics = cl_strategy.evaluator.get_all_metrics() print(f"Stored metrics: {list(all_metrics.keys())}")
def test_periodic_eval(self): model = SimpleMLP(input_size=6, hidden_size=10) model.classifier = IncrementalClassifier(model.classifier.in_features) benchmark = get_fast_benchmark() optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() curve_key = "Top1_Acc_Stream/eval_phase/train_stream/Task000" ################### # Case #1: No eval ################### # we use stream acc. because it emits a single value # for each eval loop. acc = StreamAccuracy() strategy = Naive( model, optimizer, criterion, train_epochs=2, eval_every=-1, evaluator=EvaluationPlugin(acc), ) strategy.train(benchmark.train_stream[0]) # eval is not called in this case assert len(strategy.evaluator.get_all_metrics()) == 0 ################### # Case #2: Eval at the end only and before training ################### acc = StreamAccuracy() evalp = EvaluationPlugin(acc) strategy = Naive( model, optimizer, criterion, train_epochs=2, eval_every=0, evaluator=evalp, ) strategy.train(benchmark.train_stream[0]) # eval is called once at the end of the training loop curve = strategy.evaluator.get_all_metrics()[curve_key][1] assert len(curve) == 2 ################### # Case #3: Eval after every epoch and before training ################### acc = StreamAccuracy() strategy = Naive( model, optimizer, criterion, train_epochs=2, eval_every=1, evaluator=EvaluationPlugin(acc), ) strategy.train(benchmark.train_stream[0]) curve = strategy.evaluator.get_all_metrics()[curve_key][1] assert len(curve) == 3 ################### # Case #4: Eval in iteration mode ################### acc = StreamAccuracy() strategy = Naive( model, optimizer, criterion, train_epochs=2, eval_every=100, evaluator=EvaluationPlugin(acc), peval_mode="iteration", ) strategy.train(benchmark.train_stream[0]) curve = strategy.evaluator.get_all_metrics()[curve_key][1] assert len(curve) == 5
weight_decay=HYPER_PARAMETER[mode]["weight_decay"], momentum=HYPER_PARAMETER[mode]["momentum"], ) scheduler = make_scheduler( optimizer, HYPER_PARAMETER[mode]["step_schedular_decay"], HYPER_PARAMETER[mode]["schedular_step"], ) plugin_list = [LRSchedulerPlugin(scheduler)] cl_strategy = Naive( model, optimizer, CrossEntropyLoss(), train_mb_size=HYPER_PARAMETER[mode]["batch_size"], train_epochs=num_epoch, eval_mb_size=HYPER_PARAMETER[mode]["batch_size"], evaluator=eval_plugin, device=device, plugins=plugin_list, ) # TRAINING LOOP print("Starting experiment...") results = [] print("Current input mode : ", mode) print("Current eval mode : ", eval_mode) for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) res = cl_strategy.train(experience)