def main(args): # Config device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # model model = SimpleMLP(input_size=32 * 32 * 3, num_classes=10) # CL Benchmark Creation scenario = SplitCIFAR10(n_experiences=5, return_task_id=True) train_stream = scenario.train_stream test_stream = scenario.test_stream # Prepare for training & testing optimizer = Adam(model.parameters(), lr=0.01) criterion = CrossEntropyLoss() # Choose a CL strategy strategy = Naive(model=model, optimizer=optimizer, criterion=criterion, train_mb_size=128, train_epochs=3, eval_mb_size=128, device=device) # train and test loop for train_task in train_stream: strategy.train(train_task, num_workers=0) strategy.eval(test_stream)
def evaluate_on_cifar_100( *, method_name: str, plugins: List[StrategyPlugin], tb_dir: str = str(TB_DIR), seed: int = 42, verbose: bool = False, train_epochs: int = 70, n_classes_per_batch: int = 10, start_lr: float = 2.0, lr_milestones: List[int] = None, lr_gamma: float = 0.2, ): assert not N_CLASSES % n_classes_per_batch, "n_classes should be a multiple of n_classes_per_batch" scenario = SplitCIFAR100(n_experiences=N_CLASSES // n_classes_per_batch) model = ResNet32(n_classes=N_CLASSES) tb_logger = TensorboardLogger(tb_dir + f"/cifar100_{n_classes_per_batch}/{method_name}/{seed}_{create_time_id()}") loggers = [tb_logger] if verbose: loggers.append(InteractiveLogger()) strategy = Naive( model=model, optimizer=SGD(model.parameters(), lr=2.0, weight_decay=0.00001), criterion=CrossEntropyLoss(), train_epochs=train_epochs, train_mb_size=128, device=device, plugins=plugins + [LRSchedulerPlugin(start_lr=start_lr, milestones=lr_milestones, gamma=lr_gamma)], evaluator=EvaluationPlugin( [ NormalizedStreamAccuracy(), NormalizedExperienceAccuracy(), ExperienceMeanRepresentationShift(MeanL2RepresentationShift()), ExperienceMeanRepresentationShift(MeanCosineRepresentationShift()), ], StreamConfusionMatrix( num_classes=N_CLASSES, image_creator=SortedCMImageCreator(scenario.classes_order), ), loggers=loggers, ), ) for i, train_task in enumerate(scenario.train_stream, 1): strategy.train(train_task, num_workers=0) strategy.eval(scenario.test_stream[:i]) tb_logger.writer.flush()
def test_no_errors(self): eval_plugin = EvaluationPlugin(accuracy_metrics(stream=True), loggers=None, benchmark=self.benchmark, strict_checks=True) strategy = Naive(self.model, self.optimizer, self.criterion, train_epochs=2, eval_every=0, evaluator=eval_plugin) for exp in self.benchmark.train_stream: strategy.train(exp, eval_streams=[self.benchmark.test_stream]) strategy.eval(self.benchmark.test_stream)
def evaluate_split_mnist( name: str, plugins: List[StrategyPlugin], seed: int, tensorboard_logs_dir: Union[str, Path] = str(TB_DIR), verbose: bool = False, criterion: Any = CrossEntropyLoss(), ): split_mnist = SplitMNIST(n_experiences=5, seed=seed) model = SimpleMLP(n_classes=split_mnist.n_classes, input_size=28 * 28) # model = SimpleCNN(n_channels=1, n_classes=split_mnist.n_classes) tb_logger = TensorboardLogger(tensorboard_logs_dir + f"/split_mnist/{name}/{seed}_{create_time_id()}") loggers = [tb_logger] if verbose: loggers.append(InteractiveLogger()) cl_strategy = Naive( model=model, optimizer=SGD(model.parameters(), lr=0.001, momentum=0.9), criterion=criterion, train_mb_size=32, train_epochs=2, eval_mb_size=32, device=device, plugins=plugins, evaluator=EvaluationPlugin( [ NormalizedStreamAccuracy(), NormalizedExperienceAccuracy(), ExperienceMeanRepresentationShift(MeanL2RepresentationShift()), ExperienceMeanRepresentationShift(MeanCosineRepresentationShift()), ], StreamConfusionMatrix( num_classes=split_mnist.n_classes, image_creator=SortedCMImageCreator(split_mnist.classes_order), ), loggers=loggers, ), ) for i, train_task in enumerate(split_mnist.train_stream, 1): cl_strategy.train(train_task, num_workers=0) cl_strategy.eval(split_mnist.test_stream[:i]) tb_logger.writer.flush()
def main(cuda: int): # --- CONFIG device = torch.device( f"cuda:{cuda}" if torch.cuda.is_available() else "cpu" ) # --- SCENARIO CREATION scenario = SplitCIFAR10(n_experiences=2, seed=42) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes, input_size=196608 // 64) # choose some metrics and evaluation method eval_plugin = EvaluationPlugin( accuracy_metrics(stream=True, experience=True), images_samples_metrics( on_train=True, on_eval=True, n_cols=10, n_rows=10, ), labels_repartition_metrics( # image_creator=repartition_bar_chart_image_creator, on_train=True, on_eval=True, ), loggers=[ TensorboardLogger(f"tb_data/{datetime.now()}"), InteractiveLogger(), ], ) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, Adam(model.parameters()), train_mb_size=128, train_epochs=1, eval_mb_size=128, device=device, plugins=[ReplayPlugin(mem_size=1_000)], evaluator=eval_plugin, ) # TRAINING LOOP for i, experience in enumerate(scenario.train_stream, 1): cl_strategy.train(experience) cl_strategy.eval(scenario.test_stream[:i])
def test_raise_warning(self): eval_plugin = EvaluationPlugin(accuracy_metrics(stream=True), loggers=None, benchmark=self.benchmark, strict_checks=False) strategy = Naive(self.model, self.optimizer, self.criterion, train_epochs=2, eval_every=-1, evaluator=eval_plugin) for exp in self.benchmark.train_stream: strategy.train(exp) strategy.eval(self.benchmark.test_stream) with self.assertWarns(UserWarning): strategy.eval(self.benchmark.test_stream[:2])
def main(args): # Model getter: specify dataset and depth of the network. model = pytorchcv_wrapper.resnet('cifar10', depth=20, pretrained=False) # Or get a more specific model. E.g. wide resnet, with depth 40 and growth # factor 8 for Cifar 10. # model = pytorchcv_wrapper.get_model("wrn40_8_cifar10", pretrained=False) # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") device = "cpu" # --- TRANSFORMATIONS transform = transforms.Compose([ ToTensor(), transforms.Normalize((0.491, 0.482, 0.446), (0.247, 0.243, 0.261)) ]) # --- SCENARIO CREATION cifar_train = CIFAR10(root=expanduser("~") + "/.avalanche/data/cifar10/", train=True, download=True, transform=transform) cifar_test = CIFAR10(root=expanduser("~") + "/.avalanche/data/cifar10/", train=False, download=True, transform=transform) scenario = nc_benchmark( cifar_train, cifar_test, 5, task_labels=False, seed=1234, fixed_class_order=[i for i in range(10)]) # choose some metrics and evaluation method interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), forgetting_metrics(experience=True), loggers=[interactive_logger]) # CREATE THE STRATEGY INSTANCE (Naive, with Replay) cl_strategy = Naive(model, torch.optim.SGD(model.parameters(), lr=0.01), CrossEntropyLoss(), train_mb_size=100, train_epochs=1, eval_mb_size=100, device=device, plugins=[ReplayPlugin(mem_size=1000)], evaluator=eval_plugin ) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience ", experience.current_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") n_batches = 5 # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) test_transform = transforms.Compose([ ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) # --------- # --- SCENARIO CREATION mnist_train = MNIST('./data/mnist', train=True, download=True, transform=train_transform) mnist_test = MNIST('./data/mnist', train=False, download=True, transform=test_transform) scenario = nc_scenario( mnist_train, mnist_test, n_batches, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) # choose some metrics and evaluation method interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), ExperienceForgetting(), loggers=[interactive_logger]) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, torch.optim.Adam(model.parameters(), lr=0.001), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device, plugins=[ReplayPlugin(mem_size=10000)], evaluator=eval_plugin ) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience ", experience.current_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) test_transform = transforms.Compose([ ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) # --------- # --- SCENARIO CREATION mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, transform=train_transform) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, transform=test_transform) scenario = nc_scenario( mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) eval_plugin = EvaluationPlugin( accuracy_metrics(epoch=True, experience=True, stream=True), loss_metrics(epoch=True, experience=True, stream=True), # save image should be False to appropriately view # results in Interactive Logger. # a tensor will be printed StreamConfusionMatrix(save_image=False, normalize='all'), loggers=InteractiveLogger() ) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device, evaluator=eval_plugin, plugins=[ReplayPlugin(5000)]) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def main(args): """ Last Avalanche version reference performance (online): Top1_Acc_Stream/eval_phase/test_stream = 0.9421 """ # --- DEFAULT PARAMS ONLINE DATA INCREMENTAL LEARNING nb_tasks = 5 # Can still design the data stream based on tasks epochs = 1 # All data is only seen once: Online batch_size = 10 # Only process small amount of data at a time return_task_id = False # Data incremental (task-agnostic/task-free) # TODO use data_incremental_generator, now experience=task # --- CONFIG device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- SCENARIO CREATION scenario = SplitMNIST(nb_tasks, return_task_id=return_task_id, fixed_class_order=[i for i in range(10)]) # --------- # MODEL CREATION model = SimpleMLP(num_classes=args.featsize, hidden_size=400, hidden_layers=2) # choose some metrics and evaluation method interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(experience=True, stream=True), loss_metrics(experience=True, stream=True), ExperienceForgetting(), loggers=[interactive_logger]) # CoPE PLUGIN cope = CoPEPlugin(mem_size=2000, p_size=args.featsize, n_classes=scenario.n_classes) # CREATE THE STRATEGY INSTANCE (NAIVE) WITH CoPE PLUGIN cl_strategy = Naive(model, torch.optim.SGD(model.parameters(), lr=0.01), cope.loss, # CoPE PPP-Loss train_mb_size=batch_size, train_epochs=epochs, eval_mb_size=100, device=device, plugins=[cope], evaluator=eval_plugin ) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience ", experience.current_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def test_callback_reachability(self): # Check that all the callbacks are called during # training and test loops. model = SimpleMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() scenario = self.create_scenario() plug = MockPlugin() strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu', plugins=[plug] ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] strategy.train(scenario.train_stream[0], num_workers=4) strategy.eval([scenario.test_stream[0]], num_workers=4) assert all(plug.activated)
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) test_transform = transforms.Compose( [ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # --------- # --- SCENARIO CREATION mnist_train = MNIST('./data/mnist', train=True, download=True, transform=train_transform) mnist_test = MNIST('./data/mnist', train=False, download=True, transform=test_transform) scenario = nc_scenario(mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def _test_integration(self, module, clf_name, plugins=[]): module = as_multitask(module, clf_name) module = module.to(self.device) optimizer = SGD( module.parameters(), lr=0.05, momentum=0.9, weight_decay=0.0002 ) strategy = Naive( module, optimizer, train_mb_size=32, eval_mb_size=32, device=self.device, plugins=plugins, ) for t, experience in enumerate(self.benchmark.train_stream): strategy.train(experience) strategy.eval(self.benchmark.test_stream[: t + 1])
def main(args): # Config device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # Model model = SimpleCNN(num_classes=5) # CL Benchmark Creation scenario = EndlessCLSim( scenario=args.scenario, # "Classes", "Illumination", "Weather" sequence_order=None, task_order=None, semseg=args.semseg, dataset_root=args.dataset_root, ) train_stream = scenario.train_stream test_stream = scenario.test_stream # Prepare for training & testing optimizer = Adam(model.parameters(), lr=0.001) criterion = CrossEntropyLoss() # Choose a CL strategy strategy = Naive( model=model, optimizer=optimizer, criterion=criterion, train_mb_size=64, train_epochs=3, eval_mb_size=128, device=device, ) # Train and test loop for train_task in train_stream: strategy.train(train_task, num_worker=0) strategy.eval(test_stream) return
def main(args): # Device config device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # model model = SimpleMLP(num_classes=10) # Here we show all the MNIST variation we offer in the "classic" benchmarks if args.mnist_type == "permuted": scenario = PermutedMNIST(n_experiences=5, seed=1) elif args.mnist_type == "rotated": scenario = RotatedMNIST(n_experiences=5, rotations_list=[30, 60, 90, 120, 150], seed=1) else: scenario = SplitMNIST(n_experiences=5, seed=1) # Than we can extract the parallel train and test streams train_stream = scenario.train_stream test_stream = scenario.test_stream # Prepare for training & testing optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9) criterion = CrossEntropyLoss() # Continual learning strategy with default logger cl_strategy = Naive( model, optimizer, criterion, train_mb_size=32, train_epochs=100, eval_mb_size=32, device=device, eval_every=1, plugins=[EarlyStoppingPlugin(args.patience, "test_stream")], ) # train and test loop results = [] for train_task, test_task in zip(train_stream, test_stream): print("Current Classes: ", train_task.classes_in_this_experience) cl_strategy.train(train_task, eval_streams=[test_task]) results.append(cl_strategy.eval(test_stream))
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) test_transform = transforms.Compose( [ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # --------- # --- SCENARIO CREATION mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, transform=train_transform) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, transform=test_transform) scenario = nc_benchmark(mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) # DEFINE THE EVALUATION PLUGIN AND LOGGER # The evaluation plugin manages the metrics computation. # It takes as argument a list of metrics and a list of loggers. # The evaluation plugin calls the loggers to serialize the metrics # and save them in persistent memory or print them in the standard output. # log to text file text_logger = TextLogger(open('log.txt', 'a')) # print to stdout interactive_logger = InteractiveLogger() csv_logger = CSVLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), forgetting_metrics(experience=True, stream=True), bwt_metrics(experience=True, stream=True), forward_transfer_metrics(experience=True, stream=True), cpu_usage_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), ram_usage_metrics(every=0.5, minibatch=True, epoch=True, experience=True, stream=True), gpu_usage_metrics(args.cuda, every=0.5, minibatch=True, epoch=True, experience=True, stream=True), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[interactive_logger, text_logger, csv_logger], collect_all=True) # collect all metrics (set to True by default) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=500, train_epochs=1, eval_mb_size=100, device=device, evaluator=eval_plugin, eval_every=1) # TRAINING LOOP print('Starting experiment...') results = [] for i, experience in enumerate(scenario.train_stream): print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) # train returns a dictionary containing last recorded value # for each metric. res = cl_strategy.train(experience, eval_streams=[scenario.test_stream]) print('Training completed') print('Computing accuracy on the whole test set') # test returns a dictionary with the last metric collected during # evaluation on that stream results.append(cl_strategy.eval(scenario.test_stream)) print(f"Test metrics:\n{results}") # Dict with all the metric curves, # only available when `collect_all` is True. # Each entry is a (x, metric value) tuple. # You can use this dictionary to manipulate the # metrics without avalanche. all_metrics = cl_strategy.evaluator.get_all_metrics() print(f"Stored metrics: {list(all_metrics.keys())}")
def main(): args = parser.parse_args() args.cuda = args.cuda == 'yes' args.disable_pbar = args.disable_pbar == 'yes' args.stable_sgd = args.stable_sgd == 'yes' print(f"args={vars(args)}") device = torch.device("cuda:0" if torch.cuda.is_available() and args.cuda else "cpu") print(f'Using device: {device}') # unique identifier uid = uuid.uuid4().hex if args.uid is None else args.uid now = str(datetime.datetime.now().date()) + "_" + ':'.join(str(datetime.datetime.now().time()).split(':')[:-1]) runname = 'T={}_id={}'.format(now, uid) if not args.resume else args.resume # Paths setupname = [args.strategy, args.exp_name, args.model, args.scenario] parentdir = os.path.join(args.save_path, '_'.join(setupname)) results_path = Path(os.path.join(parentdir, runname)) results_path.mkdir(parents=True, exist_ok=True) tb_log_dir = os.path.join(results_path, 'tb_run') # Group all runs # Eval results eval_metric = 'Top1_Acc_Stream/eval_phase/test_stream' eval_results_dir = results_path / eval_metric.split('/')[0] eval_results_dir.mkdir(parents=True, exist_ok=True) eval_result_files = [] # To avg over seeds seeds = [args.seed] if args.seed is not None else list(range(args.n_seeds)) for seed in seeds: # initialize seeds print("STARTING SEED {}/{}".format(seed, len(seeds) - 1)) set_seed(seed) # create scenario if args.scenario == 'smnist': inputsize = 28 * 28 scenario = SplitMNIST(n_experiences=5, return_task_id=False, seed=seed, fixed_class_order=[i for i in range(10)]) elif args.scenario == 'CIFAR10': scenario = SplitCIFAR10(n_experiences=5, return_task_id=False, seed=seed, fixed_class_order=[i for i in range(10)]) inputsize = (3, 32, 32) elif args.scenario == 'miniimgnet': scenario = SplitMiniImageNet(args.dset_rootpath, n_experiences=20, return_task_id=False, seed=seed, fixed_class_order=[i for i in range(100)]) inputsize = (3, 84, 84) else: raise ValueError("Wrong scenario name.") print(f"Scenario = {args.scenario}") if args.model == 'simple_mlp': model = MyMLP(input_size=inputsize, hidden_size=args.hs) elif args.model == 'resnet18': if not args.stable_sgd: assert args.drop_prob == 0 model = ResNet18(inputsize, scenario.n_classes, drop_prob=args.drop_prob) criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=args.lr) # Paths eval_results_file = eval_results_dir / f'seed={seed}.csv' # LOGGING tb_logger = TensorboardLogger(tb_log_dir=tb_log_dir, tb_log_exp_name=f'seed={seed}.pt') # log to Tensorboard print_logger = TextLogger() if args.disable_pbar else InteractiveLogger() # print to stdout eval_logger = EvalTextLogger(metric_filter=eval_metric, file=open(eval_results_file, 'a')) eval_result_files.append(eval_results_file) # METRICS eval_plugin = EvaluationPlugin( accuracy_metrics(experience=True, stream=True), loss_metrics(minibatch=True, experience=True), ExperienceForgetting(), # Test only StreamConfusionMatrix(num_classes=scenario.n_classes, save_image=True), # LOG OTHER STATS # timing_metrics(epoch=True, experience=False), # cpu_usage_metrics(experience=True), # DiskUsageMonitor(), # MinibatchMaxRAM(), # GpuUsageMonitor(0), loggers=[print_logger, tb_logger, eval_logger]) plugins = None if args.strategy == 'replay': plugins = [RehRevPlugin(n_total_memories=args.mem_size, mode=args.replay_mode, # STEP-BACK aversion_steps=args.aversion_steps, aversion_lr=args.aversion_lr, stable_sgd=args.stable_sgd, # Stable SGD lr_decay=args.lr_decay, init_epochs=args.init_epochs # First task epochs )] # CREATE THE STRATEGY INSTANCE (NAIVE) strategy = Naive(model, optimizer, criterion, train_epochs=args.epochs, device=device, train_mb_size=args.bs, evaluator=eval_plugin, plugins=plugins ) # train on the selected scenario with the chosen strategy print('Starting experiment...') for experience in scenario.train_stream: if experience.current_experience == args.until_task: print("CUTTING OF TRAINING AT TASK ", experience.current_experience) break else: print("Start training on step ", experience.current_experience) strategy.train(experience) print("End training on step ", experience.current_experience) print('Computing accuracy on the test set') res = strategy.eval(scenario.test_stream[:args.until_task]) # Gathered by EvalLogger final_results_file = eval_results_dir / f'seed_summary.pt' stat_summarize(eval_result_files, final_results_file) print(f"[FILE:TB-RESULTS]: {tb_log_dir}") print(f"[FILE:FINAL-RESULTS]: {final_results_file}") print("FINISHED SCRIPT")
def main(args): # Device config device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) # Intialize the model, stream and training strategy model = SimpleCNN(num_classes=10) if args.stream != "s_long": model_init = deepcopy(model) scenario = CTrL( stream_name=args.stream, save_to_disk=args.save, path=args.path, seed=10 ) train_stream = scenario.train_stream test_stream = scenario.test_stream val_stream = scenario.val_stream optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9) criterion = CrossEntropyLoss() logger = EvaluationPlugin( accuracy_metrics( minibatch=False, epoch=False, experience=True, stream=True ), loggers=[InteractiveLogger()], ) cl_strategy = Naive( model, optimizer, criterion, train_mb_size=32, device=device, train_epochs=args.max_epochs, eval_mb_size=128, evaluator=logger, plugins=[EarlyStoppingPlugin(50, "val_stream")], eval_every=5, ) # train and test loop for train_task, val_task in zip(train_stream, val_stream): cl_strategy.train(train_task, eval_streams=[val_task]) cl_strategy.eval(test_stream) transfer_mat = [] for tid in range(len(train_stream)): transfer_mat.append( logger.all_metric_results[ "Top1_Acc_Exp/eval_phase/test_stream/" f"Task00{tid}/Exp00{tid}" ][1] ) if args.stream == "s_long": res = logger.last_metric_results[ "Top1_Acc_Stream/eval_phase/" "test_stream" ] print(f"Average accuracy on S_long : {res}") else: optimizer = SGD(model_init.parameters(), lr=0.001, momentum=0.9) cl_strategy = Naive( model_init, optimizer, criterion, train_mb_size=32, device=device, train_epochs=args.max_epochs, eval_mb_size=128, plugins=[EarlyStoppingPlugin(50, "val_stream")], eval_every=5, ) cl_strategy.train(train_stream[-1]) res = cl_strategy.eval([test_stream[-1]]) acc_last_stream = transfer_mat[-1][-1] acc_last_only = res[ "Top1_Acc_Exp/eval_phase/test_stream/" "Task005/Exp-01" ] transfer_value = acc_last_stream - acc_last_only print( f"Accuracy on probe task after training on the whole " f"stream: {acc_last_stream}" ) print( f"Accuracy on probe task after trained " f"independently: {acc_last_only}" ) print(f"T({args.stream})={transfer_value}")
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) test_transform = transforms.Compose( [ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # --------- # --- SCENARIO CREATION mnist_train = MNIST('./data/mnist', train=True, download=True, transform=train_transform) mnist_test = MNIST('./data/mnist', train=False, download=True, transform=test_transform) scenario = nc_scenario(mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) # DEFINE THE EVALUATION PLUGIN AND LOGGER # The evaluation plugin manages the metrics computation. # It takes as argument a list of metrics and a list of loggers. # The evaluation plugin calls the loggers to serialize the metrics # and save them in persistent memory or print them in the standard output. # log to text file text_logger = TextLogger(open('log.txt', 'a')) # print to stdout interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin(accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), cpu_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, experience=True, stream=True), ExperienceForgetting(), loggers=[interactive_logger, text_logger]) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=500, train_epochs=1, eval_mb_size=100, device=device, evaluator=eval_plugin) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) # train returns a list of dictionaries (one for each experience). Each # dictionary stores the last value of each metric curve emitted # during training. res = cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') # test also returns a dictionary results.append(cl_strategy.eval(scenario.test_stream)) print(f"Test metrics:\n{results}") # All the metric curves (x,y values) are stored inside the evaluator # (can be disabled). You can use this dictionary to manipulate the # metrics without avalanche. all_metrics = cl_strategy.evaluator.all_metrics print(f"Stored metrics: {list(all_metrics.keys())}") mname = 'Top1_Acc_Task/Task000' print(f"{mname}: {cl_strategy.evaluator.all_metrics[mname]}")
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) test_transform = transforms.Compose( [ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # --------- # --- SCENARIO CREATION mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, transform=train_transform) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, transform=test_transform) scenario = nc_benchmark(mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) interactive_logger = InteractiveLogger() tensorboard_logger = TensorboardLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), forgetting_metrics(experience=True, stream=True), StreamConfusionMatrix(), cpu_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, experience=True, stream=True), ram_usage_metrics(every=0.5, minibatch=True, epoch=True, experience=True, stream=True), gpu_usage_metrics(args.cuda, every=0.5, minibatch=True, epoch=True, experience=True, stream=True), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[interactive_logger, tensorboard_logger]) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device, evaluator=eval_plugin) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') results.append(cl_strategy.eval(scenario.test_stream))
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- tr_ds = [ AvalancheTensorDataset( torch.randn(10, 3), torch.randint(0, 3, (10, )).tolist(), task_labels=torch.randint(0, 5, (10, )).tolist(), ) for _ in range(3) ] ts_ds = [ AvalancheTensorDataset( torch.randn(10, 3), torch.randint(0, 3, (10, )).tolist(), task_labels=torch.randint(0, 5, (10, )).tolist(), ) for _ in range(3) ] scenario = create_multi_dataset_generic_benchmark(train_datasets=tr_ds, test_datasets=ts_ds) # --------- # MODEL CREATION model = SimpleMLP(num_classes=3, input_size=3) # DEFINE THE EVALUATION PLUGIN AND LOGGER # The evaluation plugin manages the metrics computation. # It takes as argument a list of metrics and a list of loggers. # The evaluation plugin calls the loggers to serialize the metrics # and save them in persistent memory or print them in the standard output. # log to text file text_logger = TextLogger(open("log.txt", "a")) # print to stdout interactive_logger = InteractiveLogger() csv_logger = CSVLogger() eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), loss_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), forgetting_metrics(experience=True, stream=True), bwt_metrics(experience=True, stream=True), cpu_usage_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), timing_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), ram_usage_metrics(every=0.5, minibatch=True, epoch=True, experience=True, stream=True), gpu_usage_metrics( args.cuda, every=0.5, minibatch=True, epoch=True, experience=True, stream=True, ), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[interactive_logger, text_logger, csv_logger], collect_all=True, ) # collect all metrics (set to True by default) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=500, train_epochs=1, eval_mb_size=100, device=device, evaluator=eval_plugin, eval_every=1, ) # TRAINING LOOP print("Starting experiment...") results = [] for i, experience in enumerate(scenario.train_stream): print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) # train returns a dictionary containing last recorded value # for each metric. res = cl_strategy.train(experience, eval_streams=[scenario.test_stream]) print("Training completed") print("Computing accuracy on the whole test set") # test returns a dictionary with the last metric collected during # evaluation on that stream results.append(cl_strategy.eval(scenario.test_stream)) print(f"Test metrics:\n{results}") # Dict with all the metric curves, # only available when `collect_all` is True. # Each entry is a (x, metric value) tuple. # You can use this dictionary to manipulate the # metrics without avalanche. all_metrics = cl_strategy.evaluator.get_all_metrics() print(f"Stored metrics: {list(all_metrics.keys())}")
def main(args): """ Last Avalanche version reference performance (online = 1 epoch): Class-incremental (online): Top1_Acc_Stream/eval_phase/test_stream = 0.9421 Data-incremental (online: Top1_Acc_Stream/eval_phase/test_stream = 0.9309 These are reference results for a single run. """ # --- DEFAULT PARAMS ONLINE DATA INCREMENTAL LEARNING nb_tasks = 5 # Can still design the data stream based on tasks batch_size = 10 # Learning agent only has small amount of data available epochs = 1 # How many times to process each mini-batch return_task_id = False # Data incremental (task-agnostic/task-free) # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- SCENARIO CREATION n_classes = 10 task_scenario = SplitMNIST( nb_tasks, return_task_id=return_task_id, fixed_class_order=[i for i in range(n_classes)], ) # Make data incremental (one batch = one experience) scenario = data_incremental_benchmark(task_scenario, experience_size=batch_size) print( f"{scenario.n_experiences} batches in online data incremental setup.") # 6002 batches for SplitMNIST with batch size 10 # --------- # MODEL CREATION model = SimpleMLP(num_classes=args.featsize, hidden_size=400, hidden_layers=2, drop_rate=0) # choose some metrics and evaluation method logger = TextLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(experience=True, stream=True), loss_metrics(experience=False, stream=True), StreamForgetting(), loggers=[logger], benchmark=scenario, ) # CoPE PLUGIN cope = CoPEPlugin(mem_size=2000, alpha=0.99, p_size=args.featsize, n_classes=n_classes) # CREATE THE STRATEGY INSTANCE (NAIVE) WITH CoPE PLUGIN cl_strategy = Naive( model, torch.optim.SGD(model.parameters(), lr=0.01), cope.ppp_loss, # CoPE PPP-Loss train_mb_size=batch_size, train_epochs=epochs, eval_mb_size=100, device=device, plugins=[cope], evaluator=eval_plugin, ) # TRAINING LOOP print("Starting experiment...") results = [] cl_strategy.train(scenario.train_stream) print("Computing accuracy on the whole test set") results.append(cl_strategy.eval(scenario.test_stream))
evaluator=eval_plugin, device=device) else: print("Strategy is not implemented!") raise NotImplementedError # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: curr_experience = experience.current_experience print("Start of experience: ", curr_experience) print("Current Classes: ", experience.classes_in_this_experience) if (args.grow_classifier): model.classifier.adaptation(experience.dataset) print("adpated classifier") print("Classiifer:", model.classifier) # train returns a dictionary which contains all the metric values res = cl_strategy.train(experience, num_workers=4) print('Training completed') print('Computing accuracy on test set') if (not args.test_on_all): scenario.test_stream.slice_ids = list(range(curr_experience + 1)) print(scenario.test_stream.slice_ids) eval_dict = cl_strategy.eval(scenario.test_stream, num_workers=4) results.append(eval_dict)