def assert_balancing(self, policy): benchmark = get_fast_benchmark(use_task_labels=True) replay = ReplayPlugin(mem_size=100, storage_policy=policy) model = SimpleMLP(num_classes=benchmark.n_classes) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001), CrossEntropyLoss(), train_mb_size=100, train_epochs=0, eval_mb_size=100, plugins=[replay], evaluator=None, ) for exp in benchmark.train_stream: cl_strategy.train(exp) ext_mem = policy.buffer_groups ext_mem_data = policy.buffer_datasets print(list(ext_mem.keys()), [len(el) for el in ext_mem_data]) # buffer size should equal self.mem_size if data is large enough len_tot = sum([len(el) for el in ext_mem_data]) assert len_tot == policy.max_size
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") n_batches = 5 # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) test_transform = transforms.Compose([ ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) # --------- # --- SCENARIO CREATION mnist_train = MNIST('./data/mnist', train=True, download=True, transform=train_transform) mnist_test = MNIST('./data/mnist', train=False, download=True, transform=test_transform) scenario = nc_scenario( mnist_train, mnist_test, n_batches, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) # choose some metrics and evaluation method interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), ExperienceForgetting(), loggers=[interactive_logger]) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, torch.optim.Adam(model.parameters(), lr=0.001), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device, plugins=[ReplayPlugin(mem_size=10000)], evaluator=eval_plugin ) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience ", experience.current_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def test_plugins_compatibility_checks(self): model = SimpleMLP(input_size=6, hidden_size=10) benchmark = get_fast_benchmark() optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() evalp = EvaluationPlugin( loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), loggers=[InteractiveLogger()], strict_checks=None, ) strategy = Naive( model, optimizer, criterion, train_epochs=2, eval_every=-1, evaluator=evalp, plugins=[ EarlyStoppingPlugin(patience=10, val_stream_name="train") ], ) strategy.train(benchmark.train_stream[0])
def test_early_stop(self): class EarlyStopP(StrategyPlugin): def after_training_iteration(self, strategy: 'BaseStrategy', **kwargs): if strategy.mb_it == 10: strategy.stop_training() model = SimpleMLP(input_size=6, hidden_size=100) criterion = CrossEntropyLoss() optimizer = SGD(model.parameters(), lr=1) strategy = Cumulative(model, optimizer, criterion, train_mb_size=1, device=get_device(), eval_mb_size=512, train_epochs=1, evaluator=None, plugins=[EarlyStopP()]) scenario = get_fast_scenario() for train_batch_info in scenario.train_stream: strategy.train(train_batch_info) assert strategy.mb_it == 11
def main(args): # Config device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # model model = SimpleMLP(num_classes=10) # CL Benchmark Creation perm_mnist = PermutedMNIST(n_experiences=5) train_stream = perm_mnist.train_stream test_stream = perm_mnist.test_stream # Prepare for training & testing optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9) criterion = CrossEntropyLoss() # Joint training strategy joint_train = JointTraining(model, optimizer, criterion, train_mb_size=32, train_epochs=1, eval_mb_size=32, device=device) # train and test loop results = [] print("Starting training.") joint_train.train(train_stream) results.append(joint_train.eval(test_stream))
def _test_scheduler_plugin(self, gamma, milestones, base_lr, epochs, reset_lr, reset_scheduler, expected): class TestPlugin(StrategyPlugin): def __init__(self, expected_lrs): super().__init__() self.expected_lrs = expected_lrs def after_training_epoch(self, strategy, **kwargs): exp_id = strategy.training_exp_counter expected_lr = self.expected_lrs[exp_id][strategy.epoch] for group in strategy.optimizer.param_groups: assert group['lr'] == expected_lr scenario = self.create_scenario() model = SimpleMLP(input_size=6, hidden_size=10) optim = SGD(model.parameters(), lr=base_lr) lrSchedulerPlugin = LRSchedulerPlugin(MultiStepLR( optim, milestones=milestones, gamma=gamma), reset_lr=reset_lr, reset_scheduler=reset_scheduler) cl_strategy = Naive(model, optim, CrossEntropyLoss(), train_mb_size=32, train_epochs=epochs, eval_mb_size=100, plugins=[lrSchedulerPlugin, TestPlugin(expected)]) cl_strategy.train(scenario.train_stream[0]) cl_strategy.train(scenario.train_stream[1])
def test_early_stop(self): class EarlyStopP(SupervisedPlugin): def after_training_iteration( self, strategy: "SupervisedTemplate", **kwargs ): if strategy.clock.train_epoch_iterations == 10: strategy.stop_training() model = SimpleMLP(input_size=6, hidden_size=100) criterion = CrossEntropyLoss() optimizer = SGD(model.parameters(), lr=1) strategy = Cumulative( model, optimizer, criterion, train_mb_size=1, device=get_device(), eval_mb_size=512, train_epochs=1, evaluator=None, plugins=[EarlyStopP()], ) benchmark = get_fast_benchmark() for train_batch_info in benchmark.train_stream: strategy.train(train_batch_info) assert strategy.clock.train_epoch_iterations == 11
def main(args): # Config device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # model model = SimpleMLP(input_size=32 * 32 * 3, num_classes=10) # CL Benchmark Creation scenario = SplitCIFAR10(n_experiences=5, return_task_id=True) train_stream = scenario.train_stream test_stream = scenario.test_stream # Prepare for training & testing optimizer = Adam(model.parameters(), lr=0.01) criterion = CrossEntropyLoss() # Choose a CL strategy strategy = Naive(model=model, optimizer=optimizer, criterion=criterion, train_mb_size=128, train_epochs=3, eval_mb_size=128, device=device) # train and test loop for train_task in train_stream: strategy.train(train_task, num_workers=0) strategy.eval(test_stream)
def _test_replay_balanced_memory(self, storage_policy, mem_size): benchmark = get_fast_benchmark(use_task_labels=True) model = SimpleMLP(input_size=6, hidden_size=10) replayPlugin = ReplayPlugin( mem_size=mem_size, storage_policy=storage_policy ) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001), CrossEntropyLoss(), train_mb_size=32, train_epochs=1, eval_mb_size=100, plugins=[replayPlugin], ) n_seen_data = 0 for step in benchmark.train_stream: n_seen_data += len(step.dataset) mem_fill = min(mem_size, n_seen_data) cl_strategy.train(step) lengths = [] for d in replayPlugin.storage_policy.buffer_datasets: lengths.append(len(d)) self.assertEqual(sum(lengths), mem_fill) # Always fully filled
def test_dataload_batch_balancing(self): scenario = get_fast_scenario() model = SimpleMLP(input_size=6, hidden_size=10) batch_size = 32 replayPlugin = ReplayPlugin(mem_size=20) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001), CrossEntropyLoss(), train_mb_size=batch_size, train_epochs=1, eval_mb_size=100, plugins=[replayPlugin]) for step in scenario.train_stream: adapted_dataset = step.dataset dataloader = MultiTaskJoinedBatchDataLoader( adapted_dataset, AvalancheConcatDataset(replayPlugin.ext_mem.values()), oversample_small_tasks=True, num_workers=0, batch_size=batch_size, shuffle=True) for mini_batch in dataloader: lengths = [] for task_id in mini_batch.keys(): lengths.append(len(mini_batch[task_id][1])) if sum(lengths) == batch_size: difference = max(lengths) - min(lengths) self.assertLessEqual(difference, 1) self.assertLessEqual(sum(lengths), batch_size) cl_strategy.train(step)
def run_ocl_lazy_stream(experience, device): """ Runs simple naive strategy for one experience. """ model = SimpleMLP(num_classes=10).to(device) model.train() optimizer = torch.optim.SGD(model.parameters(), lr=0.01) criterion = torch.nn.CrossEntropyLoss() start = time.time() print("Running ocl_lazy_stream ...") for exp in tqdm(fixed_size_experience_split(experience, 1)): x, y, _ = exp.dataset[:] x, y = x.to(device), torch.tensor([y]).to(device) x, y = x.to(device), y.to(device) optimizer.zero_grad() pred = model(x) loss = criterion(pred, y) loss.backward() optimizer.step() end = time.time() duration = end - start return duration
def test_replay_balanced_memory(self): scenario = self.create_scenario(task_labels=True) mem_size = 25 model = SimpleMLP(input_size=6, hidden_size=10) replayPlugin = ReplayPlugin(mem_size=mem_size) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001), CrossEntropyLoss(), train_mb_size=32, train_epochs=1, eval_mb_size=100, plugins=[replayPlugin]) for step in scenario.train_stream: curr_mem_size = min(mem_size, len(step.dataset)) cl_strategy.train(step) ext_mem = replayPlugin.ext_mem lengths = [] for task_id in ext_mem.keys(): lengths.append(len(ext_mem[task_id])) self.assertEqual(sum(lengths), curr_mem_size) difference = max(lengths) - min(lengths) self.assertLessEqual(difference, 1)
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) test_transform = transforms.Compose([ ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) # --------- # --- SCENARIO CREATION mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, transform=train_transform) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, transform=test_transform) scenario = nc_scenario( mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) eval_plugin = EvaluationPlugin( accuracy_metrics(epoch=True, experience=True, stream=True), loss_metrics(epoch=True, experience=True, stream=True), # save image should be False to appropriately view # results in Interactive Logger. # a tensor will be printed StreamConfusionMatrix(save_image=False, normalize='all'), loggers=InteractiveLogger() ) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device, evaluator=eval_plugin, plugins=[ReplayPlugin(5000)]) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def main(args): """ Last Avalanche version reference performance (online): Top1_Acc_Stream/eval_phase/test_stream = 0.9421 """ # --- DEFAULT PARAMS ONLINE DATA INCREMENTAL LEARNING nb_tasks = 5 # Can still design the data stream based on tasks epochs = 1 # All data is only seen once: Online batch_size = 10 # Only process small amount of data at a time return_task_id = False # Data incremental (task-agnostic/task-free) # TODO use data_incremental_generator, now experience=task # --- CONFIG device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- SCENARIO CREATION scenario = SplitMNIST(nb_tasks, return_task_id=return_task_id, fixed_class_order=[i for i in range(10)]) # --------- # MODEL CREATION model = SimpleMLP(num_classes=args.featsize, hidden_size=400, hidden_layers=2) # choose some metrics and evaluation method interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(experience=True, stream=True), loss_metrics(experience=True, stream=True), ExperienceForgetting(), loggers=[interactive_logger]) # CoPE PLUGIN cope = CoPEPlugin(mem_size=2000, p_size=args.featsize, n_classes=scenario.n_classes) # CREATE THE STRATEGY INSTANCE (NAIVE) WITH CoPE PLUGIN cl_strategy = Naive(model, torch.optim.SGD(model.parameters(), lr=0.01), cope.loss, # CoPE PPP-Loss train_mb_size=batch_size, train_epochs=epochs, eval_mb_size=100, device=device, plugins=[cope], evaluator=eval_plugin ) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience ", experience.current_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def main(args): model = SimpleMLP(hidden_size=args.hs) optimizer = torch.optim.SGD(model.parameters(), lr=args.lr) criterion = torch.nn.CrossEntropyLoss() # check if selected GPU is available or use CPU assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0." device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) print(f"Using device: {device}") # create Permuted MNIST scenario scenario = PermutedMNIST(n_experiences=4) interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, experience=True, stream=True ), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), forgetting_metrics(experience=True), loggers=[interactive_logger], ) # create strategy assert ( len(args.lambda_e) == 1 or len(args.lambda_e) == 5 ), "Lambda_e must be a non-empty list." lambda_e = args.lambda_e[0] if len(args.lambda_e) == 1 else args.lambda_e strategy = LFL( model, optimizer, criterion, lambda_e=lambda_e, train_epochs=args.epochs, device=device, train_mb_size=args.minibatch_size, evaluator=eval_plugin, ) # train on the selected scenario with the chosen strategy print("Starting experiment...") results = [] for train_batch_info in scenario.train_stream: print( "Start training on experience ", train_batch_info.current_experience ) strategy.train(train_batch_info, num_workers=0) print( "End training on experience ", train_batch_info.current_experience ) print("Computing accuracy on the test set") results.append(strategy.eval(scenario.test_stream[:]))
def test_incremental_classifier(self): model = SimpleMLP(input_size=6, hidden_size=10) model.classifier = IncrementalClassifier(in_features=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() scenario = self.scenario strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu') strategy.evaluator.loggers = [TextLogger(sys.stdout)] print("Current Classes: ", scenario.train_stream[0].classes_in_this_experience) print("Current Classes: ", scenario.train_stream[4].classes_in_this_experience) # train on first task strategy.train(scenario.train_stream[0]) w_ptr = model.classifier.classifier.weight.data_ptr() b_ptr = model.classifier.classifier.bias.data_ptr() opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group['params'] ] # classifier params should be optimized assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # train again on the same task. strategy.train(scenario.train_stream[0]) # parameters should not change. assert w_ptr == model.classifier.classifier.weight.data_ptr() assert b_ptr == model.classifier.classifier.bias.data_ptr() # the same classifier params should still be optimized assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # update classifier with new classes. old_w_ptr, old_b_ptr = w_ptr, b_ptr strategy.train(scenario.train_stream[4]) opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group['params'] ] new_w_ptr = model.classifier.classifier.weight.data_ptr() new_b_ptr = model.classifier.classifier.bias.data_ptr() # weights should change. assert old_w_ptr != new_w_ptr assert old_b_ptr != new_b_ptr # Old params should not be optimized. New params should be optimized. assert old_w_ptr not in opt_params_ptrs assert old_b_ptr not in opt_params_ptrs assert new_w_ptr in opt_params_ptrs assert new_b_ptr in opt_params_ptrs
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) test_transform = transforms.Compose( [ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # --------- # --- SCENARIO CREATION mnist_train = MNIST('./data/mnist', train=True, download=True, transform=train_transform) mnist_test = MNIST('./data/mnist', train=False, download=True, transform=test_transform) scenario = nc_scenario(mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def main(args): model = SimpleMLP(hidden_size=args.hs) optimizer = torch.optim.SGD(model.parameters(), lr=args.lr) criterion = torch.nn.CrossEntropyLoss() # check if selected GPU is available or use CPU assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0." device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") print(f'Using device: {device}') # create scenario if args.scenario == 'pmnist': scenario = PermutedMNIST(n_experiences=args.permutations) elif args.scenario == 'smnist': scenario = SplitMNIST(n_experiences=5, return_task_id=False) else: raise ValueError("Wrong scenario name. Allowed pmnist, smnist.") # choose some metrics and evaluation method interactive_logger = InteractiveLogger() text_logger = TextLogger(open('log.txt', 'a')) eval_plugin = EvaluationPlugin(accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), ExperienceForgetting(), loggers=[interactive_logger]) # create strategy strategy = EWC(model, optimizer, criterion, args.ewc_lambda, args.ewc_mode, decay_factor=args.decay_factor, train_epochs=args.epochs, device=device, train_mb_size=args.minibatch_size, evaluator=eval_plugin) # train on the selected scenario with the chosen strategy print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start training on experience ", experience.current_experience) strategy.train(experience) print("End training on experience", experience.current_experience) print('Computing accuracy on the test set') results.append(strategy.eval(scenario.test_stream[:]))
def main(args): model = SimpleMLP(hidden_size=args.hs) optimizer = torch.optim.SGD(model.parameters(), lr=args.lr) criterion = torch.nn.CrossEntropyLoss() # check if selected GPU is available or use CPU assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0." device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") print(f'Using device: {device}') # create split scenario scenario = SplitMNIST(n_experiences=5, return_task_id=False) interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin(accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), ExperienceForgetting(), loggers=[interactive_logger]) # create strategy assert len(args.lwf_alpha) == 1 or len(args.lwf_alpha) == 5,\ 'Alpha must be a non-empty list.' lwf_alpha = args.lwf_alpha[0] if len( args.lwf_alpha) == 1 else args.lwf_alpha strategy = LwF(model, optimizer, criterion, alpha=lwf_alpha, temperature=args.softmax_temperature, train_epochs=args.epochs, device=device, train_mb_size=args.minibatch_size, evaluator=eval_plugin) # train on the selected scenario with the chosen strategy print('Starting experiment...') results = [] for train_batch_info in scenario.train_stream: print("Start training on experience ", train_batch_info.current_experience) strategy.train(train_batch_info, num_workers=4) print("End training on experience ", train_batch_info.current_experience) print('Computing accuracy on the test set') results.append(strategy.eval(scenario.test_stream[:]))
def get_model(self, fast_test=False): if fast_test: model = SimpleMLP(input_size=6, hidden_size=10) # model.classifier = IncrementalClassifier( # model.classifier.in_features) return model else: model = SimpleMLP() # model.classifier = IncrementalClassifier( # model.classifier.in_features) return model
def test_periodic_eval(self): model = SimpleMLP(input_size=6, hidden_size=10) scenario = get_fast_scenario() optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() curve_key = 'Top1_Acc_Stream/eval_phase/train_stream' ################### # Case #1: No eval ################### # we use stream acc. because it emits a single value # for each eval loop. acc = StreamAccuracy() strategy = Naive(model, optimizer, criterion, train_epochs=2, eval_every=-1, evaluator=EvaluationPlugin(acc)) strategy.train(scenario.train_stream[0]) # eval is not called in this case assert len(strategy.evaluator.all_metrics) == 0 ################### # Case #2: Eval at the end only ################### acc = StreamAccuracy() strategy = Naive(model, optimizer, criterion, train_epochs=2, eval_every=0, evaluator=EvaluationPlugin(acc)) strategy.train(scenario.train_stream[0]) # eval is called once at the end of the training loop curve = strategy.evaluator.all_metrics[curve_key][1] assert len(curve) == 1 ################### # Case #3: Eval after every epoch ################### acc = StreamAccuracy() strategy = Naive(model, optimizer, criterion, train_epochs=2, eval_every=1, evaluator=EvaluationPlugin(acc)) strategy.train(scenario.train_stream[0]) # eval is called after every epoch + the end of the training loop curve = strategy.evaluator.all_metrics[curve_key][1] assert len(curve) == 3
def test_dataload_reinit(self): scenario = get_fast_scenario() model = SimpleMLP(input_size=6, hidden_size=10) replayPlugin = ReplayPlugin(mem_size=5) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001), CrossEntropyLoss(), train_mb_size=16, train_epochs=1, eval_mb_size=16, plugins=[replayPlugin] ) for step in scenario.train_stream[:2]: cl_strategy.train(step)
def test_initialisation(self): module = SimpleMLP() module = module.to(self.device) old_classifier_weight = torch.clone(module.classifier.weight) old_classifier_bias = torch.clone(module.classifier.bias) module = as_multitask(module, "classifier") module = module.to(self.device) new_classifier_weight = torch.clone( module.classifier.classifiers["0"].classifier.weight) new_classifier_bias = torch.clone( module.classifier.classifiers["0"].classifier.bias) self.assertTrue( torch.equal(old_classifier_weight, new_classifier_weight)) self.assertTrue(torch.equal(old_classifier_bias, new_classifier_bias))
def main(cuda: int): # --- CONFIG device = torch.device( f"cuda:{cuda}" if torch.cuda.is_available() else "cpu" ) # --- SCENARIO CREATION scenario = SplitCIFAR10(n_experiences=2, seed=42) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes, input_size=196608 // 64) # choose some metrics and evaluation method eval_plugin = EvaluationPlugin( accuracy_metrics(stream=True, experience=True), images_samples_metrics( on_train=True, on_eval=True, n_cols=10, n_rows=10, ), labels_repartition_metrics( # image_creator=repartition_bar_chart_image_creator, on_train=True, on_eval=True, ), loggers=[ TensorboardLogger(f"tb_data/{datetime.now()}"), InteractiveLogger(), ], ) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, Adam(model.parameters()), train_mb_size=128, train_epochs=1, eval_mb_size=128, device=device, plugins=[ReplayPlugin(mem_size=1_000)], evaluator=eval_plugin, ) # TRAINING LOOP for i, experience in enumerate(scenario.train_stream, 1): cl_strategy.train(experience) cl_strategy.eval(scenario.test_stream[:i])
def main(args): # Device config device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) # model model = SimpleMLP(num_classes=10) # Here we show all the MNIST variation we offer in the "classic" benchmarks if args.mnist_type == "permuted": scenario = PermutedMNIST(n_experiences=5, seed=1) elif args.mnist_type == "rotated": scenario = RotatedMNIST( n_experiences=5, rotations_list=[30, 60, 90, 120, 150], seed=1 ) else: scenario = SplitMNIST(n_experiences=5, seed=1) # Than we can extract the parallel train and test streams train_stream = scenario.train_stream test_stream = scenario.test_stream # Prepare for training & testing optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9) criterion = CrossEntropyLoss() # Continual learning strategy with default logger cl_strategy = Naive( model, optimizer, criterion, train_mb_size=32, train_epochs=100, eval_mb_size=32, device=device, eval_every=1, plugins=[EarlyStoppingPlugin(args.patience, "test_stream")], ) # train and test loop results = [] for train_task, test_task in zip(train_stream, test_stream): print("Current Classes: ", train_task.classes_in_this_experience) cl_strategy.train(train_task, eval_streams=[test_task]) results.append(cl_strategy.eval(test_stream))
def test_optimizer_update(self): model = SimpleMLP() optimizer = SGD(model.parameters(), lr=1e-3) strategy = Naive(model, optimizer, None) # check add_param_group p = torch.nn.Parameter(torch.zeros(10, 10)) strategy.add_new_params_to_optimizer(p) assert self._is_param_in_optimizer(p, strategy.optimizer) # check new_param is in optimizer # check old_param is NOT in optimizer p_new = torch.nn.Parameter(torch.zeros(10, 10)) strategy.update_optimizer([p], [p_new]) assert self._is_param_in_optimizer(p_new, strategy.optimizer) assert not self._is_param_in_optimizer(p, strategy.optimizer)
def test_outputs(self): modules = [ (SimpleMLP(input_size=32 * 32 * 3), "classifier"), (SimpleCNN(), "classifier"), ] for m in modules: self._test_outputs(*m)
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --- SCENARIO CREATION scenario = SplitMNIST(n_experiences=10, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) # choose some metrics and evaluation method interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), forgetting_metrics(experience=True), loggers=[interactive_logger], ) # CREATE THE STRATEGY INSTANCE (GenerativeReplay) cl_strategy = GenerativeReplay( model, torch.optim.Adam(model.parameters(), lr=0.001), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device, evaluator=eval_plugin, ) # TRAINING LOOP print("Starting experiment...") results = [] for experience in scenario.train_stream: print("Start of experience ", experience.current_experience) cl_strategy.train(experience) print("Training completed") print("Computing accuracy on the whole test set") results.append(cl_strategy.eval(scenario.test_stream))
def test_callback_reachability(self): # Check that all the callbacks are called during # training and test loops. model = SimpleMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() scenario = self.create_scenario() plug = MockPlugin() strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu', plugins=[plug] ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] strategy.train(scenario.train_stream[0], num_workers=4) strategy.eval([scenario.test_stream[0]], num_workers=4) assert all(plug.activated)
def test_multihead_optimizer_update(self): # Check if the optimizer is updated correctly # when heads are created and updated. model = SimpleMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() scenario = self.create_scenario() plug = MultiHeadPlugin(model, 'classifier') strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu', plugins=[plug]) strategy.evaluator.loggers = [TextLogger(sys.stdout)] print("Current Classes: ", scenario.train_stream[0].classes_in_this_experience) print("Current Classes: ", scenario.train_stream[4].classes_in_this_experience) # head creation strategy.train(scenario.train_stream[0]) w_ptr = model.classifier.weight.data_ptr() b_ptr = model.classifier.bias.data_ptr() opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group['params'] ] assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # head update strategy.train(scenario.train_stream[4]) w_ptr_new = model.classifier.weight.data_ptr() b_ptr_new = model.classifier.bias.data_ptr() opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group['params'] ] assert w_ptr not in opt_params_ptrs assert b_ptr not in opt_params_ptrs assert w_ptr_new in opt_params_ptrs assert b_ptr_new in opt_params_ptrs