def test_joint(self): class JointSTestPlugin(SupervisedPlugin): def __init__(self, benchmark): super().__init__() self.benchmark = benchmark def after_train_dataset_adaptation( self, strategy: "SupervisedTemplate", **kwargs ): """ Check that the dataset used for training contains the correct number of samples. """ cum_len = sum( [len(exp.dataset) for exp in self.benchmark.train_stream] ) assert len(strategy.adapted_dataset) == cum_len # SIT scenario model, optimizer, criterion, my_nc_benchmark = self.init_sit() strategy = JointTraining( model, optimizer, criterion, train_mb_size=64, device=self.device, eval_mb_size=50, train_epochs=2, plugins=[JointSTestPlugin(my_nc_benchmark)], ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] strategy.train(my_nc_benchmark.train_stream) # MT scenario my_nc_benchmark = self.load_benchmark(use_task_labels=True) strategy = JointTraining( model, optimizer, criterion, train_mb_size=64, device=self.device, eval_mb_size=50, train_epochs=2, plugins=[JointSTestPlugin(my_nc_benchmark)], ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] strategy.train(my_nc_benchmark.train_stream) # Raise error when retraining self.assertRaises( AlreadyTrainedError, lambda: strategy.train(my_nc_benchmark.train_stream), )
def test_multihead_head_selection(self): # Check if the optimizer is updated correctly # when heads are created and updated. model = MultiHeadClassifier(in_features=6) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() benchmark = get_fast_benchmark(use_task_labels=True, shuffle=False) strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu') strategy.evaluator.loggers = [TextLogger(sys.stdout)] # initialize head strategy.train(benchmark.train_stream[0]) strategy.train(benchmark.train_stream[4]) # create models with fixed head model_t0 = model.classifiers['0'] model_t4 = model.classifiers['4'] # check head task0 for x, y, t in DataLoader(benchmark.train_stream[0].dataset): y_mh = model(x, t) y_t = model_t0(x) assert ((y_mh - y_t) ** 2).sum() < 1.e-7 break # check head task4 for x, y, t in DataLoader(benchmark.train_stream[4].dataset): y_mh = model(x, t) y_t = model_t4(x) assert ((y_mh - y_t) ** 2).sum() < 1.e-7 break
def test_incremental_classifier_weight_update(self): model = IncrementalClassifier(in_features=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() scenario = self.scenario strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu') strategy.evaluator.loggers = [TextLogger(sys.stdout)] # train on first task w_old = model.classifier.weight.clone() b_old = model.classifier.bias.clone() # adaptation. Increase number of classes dataset = scenario.train_stream[4].dataset model.adaptation(dataset) w_new = model.classifier.weight.clone() b_new = model.classifier.bias.clone() # old weights should be copied correctly. assert torch.equal(w_old, w_new[:w_old.shape[0]]) assert torch.equal(b_old, b_new[:w_old.shape[0]]) # shape should be correct. assert w_new.shape[0] == max(dataset.targets) + 1 assert b_new.shape[0] == max(dataset.targets) + 1
def test_incremental_classifier(self): model = SimpleMLP(input_size=6, hidden_size=10) model.classifier = IncrementalClassifier(in_features=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() scenario = self.scenario strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu') strategy.evaluator.loggers = [TextLogger(sys.stdout)] print("Current Classes: ", scenario.train_stream[0].classes_in_this_experience) print("Current Classes: ", scenario.train_stream[4].classes_in_this_experience) # train on first task strategy.train(scenario.train_stream[0]) w_ptr = model.classifier.classifier.weight.data_ptr() b_ptr = model.classifier.classifier.bias.data_ptr() opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group['params'] ] # classifier params should be optimized assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # train again on the same task. strategy.train(scenario.train_stream[0]) # parameters should not change. assert w_ptr == model.classifier.classifier.weight.data_ptr() assert b_ptr == model.classifier.classifier.bias.data_ptr() # the same classifier params should still be optimized assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # update classifier with new classes. old_w_ptr, old_b_ptr = w_ptr, b_ptr strategy.train(scenario.train_stream[4]) opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group['params'] ] new_w_ptr = model.classifier.classifier.weight.data_ptr() new_b_ptr = model.classifier.classifier.bias.data_ptr() # weights should change. assert old_w_ptr != new_w_ptr assert old_b_ptr != new_b_ptr # Old params should not be optimized. New params should be optimized. assert old_w_ptr not in opt_params_ptrs assert old_b_ptr not in opt_params_ptrs assert new_w_ptr in opt_params_ptrs assert new_b_ptr in opt_params_ptrs
def test_multihead_head_creation(self): # Check if the optimizer is updated correctly # when heads are created and updated. model = MTSimpleMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() benchmark = get_fast_benchmark(use_task_labels=True, shuffle=False) strategy = Naive( model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device="cpu", ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] print( "Current Classes: ", benchmark.train_stream[4].classes_in_this_experience, ) print( "Current Classes: ", benchmark.train_stream[0].classes_in_this_experience, ) # head creation strategy.train(benchmark.train_stream[0]) w_ptr = model.classifier.classifiers["0"].classifier.weight.data_ptr() b_ptr = model.classifier.classifiers["0"].classifier.bias.data_ptr() opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group["params"] ] assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # head update strategy.train(benchmark.train_stream[4]) w_ptr_t0 = model.classifier.classifiers[ "0"].classifier.weight.data_ptr() b_ptr_t0 = model.classifier.classifiers["0"].classifier.bias.data_ptr() w_ptr_new = model.classifier.classifiers[ "4"].classifier.weight.data_ptr() b_ptr_new = model.classifier.classifiers["4"].classifier.bias.data_ptr( ) opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group["params"] ] assert w_ptr not in opt_params_ptrs # head0 has been updated assert b_ptr not in opt_params_ptrs # head0 has been updated assert w_ptr_t0 in opt_params_ptrs assert b_ptr_t0 in opt_params_ptrs assert w_ptr_new in opt_params_ptrs assert b_ptr_new in opt_params_ptrs
def main(args): model = SimpleMLP(hidden_size=args.hs) optimizer = torch.optim.SGD(model.parameters(), lr=args.lr) criterion = torch.nn.CrossEntropyLoss() # check if selected GPU is available or use CPU assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0." device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") print(f'Using device: {device}') # create scenario if args.scenario == 'pmnist': scenario = PermutedMNIST(n_experiences=args.permutations) elif args.scenario == 'smnist': scenario = SplitMNIST(n_experiences=5, return_task_id=False) else: raise ValueError("Wrong scenario name. Allowed pmnist, smnist.") # choose some metrics and evaluation method interactive_logger = InteractiveLogger() text_logger = TextLogger(open('log.txt', 'a')) eval_plugin = EvaluationPlugin(accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), ExperienceForgetting(), loggers=[interactive_logger]) # create strategy strategy = EWC(model, optimizer, criterion, args.ewc_lambda, args.ewc_mode, decay_factor=args.decay_factor, train_epochs=args.epochs, device=device, train_mb_size=args.minibatch_size, evaluator=eval_plugin) # train on the selected scenario with the chosen strategy print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start training on experience ", experience.current_experience) strategy.train(experience) print("End training on experience", experience.current_experience) print('Computing accuracy on the test set') results.append(strategy.eval(scenario.test_stream[:]))
def run_strategy(self, benchmark, cl_strategy): print("Starting experiment...") cl_strategy.evaluator.loggers = [TextLogger(sys.stdout)] results = [] for train_batch_info in benchmark.train_stream: print("Start of experience ", train_batch_info.current_experience) cl_strategy.train(train_batch_info, num_workers=0) print("Training completed") print("Computing accuracy on the current test set") results.append(cl_strategy.eval(benchmark.test_stream[:]))
def run_strategy(self, scenario, cl_strategy): print('Starting experiment...') cl_strategy.evaluator.loggers = [TextLogger(sys.stdout)] results = [] for train_batch_info in scenario.train_stream: print("Start of experience ", train_batch_info.current_experience) cl_strategy.train(train_batch_info) print('Training completed') print('Computing accuracy on the current test set') results.append(cl_strategy.eval(scenario.test_stream[:]))
def test_multihead_optimizer_update(self): # Check if the optimizer is updated correctly # when heads are created and updated. model = SimpleMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() scenario = self.create_scenario() plug = MultiHeadPlugin(model, 'classifier') strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu', plugins=[plug]) strategy.evaluator.loggers = [TextLogger(sys.stdout)] print("Current Classes: ", scenario.train_stream[0].classes_in_this_experience) print("Current Classes: ", scenario.train_stream[4].classes_in_this_experience) # head creation strategy.train(scenario.train_stream[0]) w_ptr = model.classifier.weight.data_ptr() b_ptr = model.classifier.bias.data_ptr() opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group['params'] ] assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # head update strategy.train(scenario.train_stream[4]) w_ptr_new = model.classifier.weight.data_ptr() b_ptr_new = model.classifier.bias.data_ptr() opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group['params'] ] assert w_ptr not in opt_params_ptrs assert b_ptr not in opt_params_ptrs assert w_ptr_new in opt_params_ptrs assert b_ptr_new in opt_params_ptrs
def test_callback_reachability(self): # Check that all the callbacks are called during # training and test loops. model = SimpleMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() scenario = self.create_scenario() plug = MockPlugin() strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu', plugins=[plug] ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] strategy.train(scenario.train_stream[0], num_workers=4) strategy.eval([scenario.test_stream[0]], num_workers=4) assert all(plug.activated)
def test_callback_reachability(self): # Check that all the callbacks are called during # training and test loops. model = _PlainMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() benchmark = PluginTests.create_benchmark() plug = MockPlugin() strategy = Naive( model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device="cpu", plugins=[plug], ) strategy.evaluator.loggers = [TextLogger(sys.stdout)] strategy.train(benchmark.train_stream[0], num_workers=0) strategy.eval([benchmark.test_stream[0]], num_workers=0) assert all(plug.activated)
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) test_transform = transforms.Compose( [ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # --------- # --- SCENARIO CREATION mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=True, download=True, transform=train_transform) mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/", train=False, download=True, transform=test_transform) scenario = nc_benchmark(mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) # DEFINE THE EVALUATION PLUGIN AND LOGGER # The evaluation plugin manages the metrics computation. # It takes as argument a list of metrics and a list of loggers. # The evaluation plugin calls the loggers to serialize the metrics # and save them in persistent memory or print them in the standard output. # log to text file text_logger = TextLogger(open('log.txt', 'a')) # print to stdout interactive_logger = InteractiveLogger() csv_logger = CSVLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), forgetting_metrics(experience=True, stream=True), bwt_metrics(experience=True, stream=True), forward_transfer_metrics(experience=True, stream=True), cpu_usage_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), ram_usage_metrics(every=0.5, minibatch=True, epoch=True, experience=True, stream=True), gpu_usage_metrics(args.cuda, every=0.5, minibatch=True, epoch=True, experience=True, stream=True), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[interactive_logger, text_logger, csv_logger], collect_all=True) # collect all metrics (set to True by default) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=500, train_epochs=1, eval_mb_size=100, device=device, evaluator=eval_plugin, eval_every=1) # TRAINING LOOP print('Starting experiment...') results = [] for i, experience in enumerate(scenario.train_stream): print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) # train returns a dictionary containing last recorded value # for each metric. res = cl_strategy.train(experience, eval_streams=[scenario.test_stream]) print('Training completed') print('Computing accuracy on the whole test set') # test returns a dictionary with the last metric collected during # evaluation on that stream results.append(cl_strategy.eval(scenario.test_stream)) print(f"Test metrics:\n{results}") # Dict with all the metric curves, # only available when `collect_all` is True. # Each entry is a (x, metric value) tuple. # You can use this dictionary to manipulate the # metrics without avalanche. all_metrics = cl_strategy.evaluator.get_all_metrics() print(f"Stored metrics: {list(all_metrics.keys())}")
generic_scenario = tensor_scenario( train_data_x=dataset[0], train_data_y=dataset[1], test_data_x=dataset[2], test_data_y=dataset[3], task_labels=[0 for key in task_order_list[i].keys() ], # shouldn't provide task ID for inference ) # log to Tensorboard tb_logger = TensorboardLogger( f"./tb_data/{cur_time}_CNN1D_0inTask{task_order}/") # log to text file text_logger = TextLogger( open(f"./logs/{cur_time}_CNN1D_0inTask{task_order}.txt", "w+")) # print to stdout interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(epoch=True, epoch_running=True), ExperienceForgetting(), cpu_usage_metrics(experience=True), StreamConfusionMatrix(num_classes=2, save_image=False), disk_usage_metrics(minibatch=True,
generic_scenario = tensor_scenario( train_data_x=dataset[0], train_data_y=dataset[1], test_data_x=dataset[2], test_data_y=dataset[3], task_labels=[ 0 for key in task_order_list[0].keys() ], # shouldn't provide task ID for inference ) # log to Tensorboard tb_logger = TensorboardLogger(f"./tb_data/{cur_time}-SimpleMLP/") # log to text file text_logger = TextLogger(open(f"./logs/{cur_time}-SimpleMLP.txt", "w+")) # print to stdout interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(epoch=True, epoch_running=True), ExperienceForgetting(), cpu_usage_metrics(experience=True), StreamConfusionMatrix(num_classes=2, save_image=False), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), loggers=[interactive_logger, text_logger, tb_logger], )
def main(args): # --- CONFIG device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) # --------- tr_ds = [ AvalancheTensorDataset( torch.randn(10, 3), torch.randint(0, 3, (10,)).tolist(), task_labels=torch.randint(0, 5, (10,)).tolist(), ) for _ in range(3) ] ts_ds = [ AvalancheTensorDataset( torch.randn(10, 3), torch.randint(0, 3, (10,)).tolist(), task_labels=torch.randint(0, 5, (10,)).tolist(), ) for _ in range(3) ] scenario = create_multi_dataset_generic_benchmark( train_datasets=tr_ds, test_datasets=ts_ds ) # --------- # MODEL CREATION model = SimpleMLP(num_classes=3, input_size=3) # DEFINE THE EVALUATION PLUGIN AND LOGGER # The evaluation plugin manages the metrics computation. # It takes as argument a list of metrics and a list of loggers. # The evaluation plugin calls the loggers to serialize the metrics # and save them in persistent memory or print them in the standard output. # log to text file text_logger = TextLogger(open("log.txt", "a")) # print to stdout interactive_logger = InteractiveLogger() csv_logger = CSVLogger() eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), loss_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), forgetting_metrics(experience=True, stream=True), bwt_metrics(experience=True, stream=True), cpu_usage_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), timing_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), ram_usage_metrics( every=0.5, minibatch=True, epoch=True, experience=True, stream=True ), gpu_usage_metrics( args.cuda, every=0.5, minibatch=True, epoch=True, experience=True, stream=True, ), disk_usage_metrics( minibatch=True, epoch=True, experience=True, stream=True ), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[interactive_logger, text_logger, csv_logger], collect_all=True, ) # collect all metrics (set to True by default) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive( model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=500, train_epochs=1, eval_mb_size=100, device=device, evaluator=eval_plugin, eval_every=1, ) # TRAINING LOOP print("Starting experiment...") results = [] for i, experience in enumerate(scenario.train_stream): print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) # train returns a dictionary containing last recorded value # for each metric. res = cl_strategy.train(experience, eval_streams=[scenario.test_stream]) print("Training completed") print("Computing accuracy on the whole test set") # test returns a dictionary with the last metric collected during # evaluation on that stream results.append(cl_strategy.eval(scenario.test_stream)) print(f"Test metrics:\n{results}") # Dict with all the metric curves, # only available when `collect_all` is True. # Each entry is a (x, metric value) tuple. # You can use this dictionary to manipulate the # metrics without avalanche. all_metrics = cl_strategy.evaluator.get_all_metrics() print(f"Stored metrics: {list(all_metrics.keys())}")
def setUpClass(cls) -> None: torch.manual_seed(0) np.random.seed(0) random.seed(0) n_samples_per_class = 100 dataset = make_classification(n_samples=6 * n_samples_per_class, n_classes=6, n_features=4, n_informative=4, n_redundant=0) X = torch.from_numpy(dataset[0]).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split(X, y, train_size=0.5, shuffle=True, stratify=y) tr_d = TensorDataset(train_X, train_y) ts_d = TensorDataset(test_X, test_y) benchmark = nc_benchmark(train_dataset=tr_d, test_dataset=ts_d, n_experiences=3, task_labels=True, shuffle=False, seed=0) model = SimpleMLP(input_size=4, num_classes=benchmark.n_classes) f = open('log.txt', 'w') text_logger = TextLogger(f) eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), forgetting_metrics(experience=True, stream=True, task=True), confusion_matrix_metrics(num_classes=6, save_image=False, normalize='all', stream=True), bwt_metrics(experience=True, stream=True, task=True), cpu_usage_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), ram_usage_metrics(every=0.5, minibatch=True, epoch=True, experience=True, stream=True), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[text_logger], collect_all=True) # collect all metrics (set to True by default) cl_strategy = BaseStrategy(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=10, train_epochs=2, eval_mb_size=10, device=DEVICE, evaluator=eval_plugin, eval_every=1) for i, experience in enumerate(benchmark.train_stream): cl_strategy.train(experience, eval_streams=[benchmark.test_stream[i]], shuffle=False) cl_strategy.eval(benchmark.test_stream) cls.all_metrics = cl_strategy.evaluator.get_all_metrics() f.close() # with open(os.path.join(pathlib.Path(__file__).parent.absolute(), # 'target_metrics', # 'mt.pickle'), 'wb') as f: # pickle.dump(dict(cls.all_metrics), f, # protocol=pickle.HIGHEST_PROTOCOL) with open( os.path.join( pathlib.Path(__file__).parent.absolute(), 'target_metrics', 'mt.pickle'), 'rb') as f: cls.ref = pickle.load(f)
def setUpClass(cls) -> None: torch.manual_seed(0) np.random.seed(0) random.seed(0) n_samples_per_class = 100 datasets = [] for i in range(3): dataset = make_classification(n_samples=3 * n_samples_per_class, n_classes=3, n_features=3, n_informative=3, n_redundant=0) X = torch.from_numpy(dataset[0]).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split(X, y, train_size=0.5, shuffle=True, stratify=y) datasets.append((train_X, train_y, test_X, test_y)) tr_ds = [ AvalancheTensorDataset( tr_X, tr_y, dataset_type=AvalancheDatasetType.CLASSIFICATION, task_labels=torch.randint(0, 3, (150, )).tolist()) for tr_X, tr_y, _, _ in datasets ] ts_ds = [ AvalancheTensorDataset( ts_X, ts_y, dataset_type=AvalancheDatasetType.CLASSIFICATION, task_labels=torch.randint(0, 3, (150, )).tolist()) for _, _, ts_X, ts_y in datasets ] benchmark = dataset_benchmark(train_datasets=tr_ds, test_datasets=ts_ds) model = SimpleMLP(num_classes=3, input_size=3) f = open('log.txt', 'w') text_logger = TextLogger(f) eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, trained_experience=True), loss_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), forgetting_metrics(experience=True, stream=True), confusion_matrix_metrics(num_classes=3, save_image=False, normalize='all', stream=True), bwt_metrics(experience=True, stream=True), forward_transfer_metrics(experience=True, stream=True), cpu_usage_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), ram_usage_metrics(every=0.5, minibatch=True, epoch=True, experience=True, stream=True), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[text_logger], collect_all=True) # collect all metrics (set to True by default) cl_strategy = BaseStrategy(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=2, train_epochs=2, eval_mb_size=2, device=DEVICE, evaluator=eval_plugin, eval_every=1) for i, experience in enumerate(benchmark.train_stream): cl_strategy.train(experience, eval_streams=[benchmark.test_stream], shuffle=False) cl_strategy.eval(benchmark.test_stream) cls.all_metrics = cl_strategy.evaluator.get_all_metrics() f.close() # # Uncomment me to regenerate the reference metrics. Make sure # # the old tests were passing for all unchanged metrics # with open(os.path.join(pathlib.Path(__file__).parent.absolute(), # 'target_metrics', # 'tpp.pickle'), 'wb') as f: # pickle.dump(dict(cls.all_metrics), f, # protocol=4) with open( os.path.join( pathlib.Path(__file__).parent.absolute(), 'target_metrics', 'tpp.pickle'), 'rb') as f: cls.ref = pickle.load(f)
def main(args): """ Last Avalanche version reference performance (online = 1 epoch): Class-incremental (online): Top1_Acc_Stream/eval_phase/test_stream = 0.9421 Data-incremental (online: Top1_Acc_Stream/eval_phase/test_stream = 0.9309 These are reference results for a single run. """ # --- DEFAULT PARAMS ONLINE DATA INCREMENTAL LEARNING nb_tasks = 5 # Can still design the data stream based on tasks batch_size = 10 # Learning agent only has small amount of data available epochs = 1 # How many times to process each mini-batch return_task_id = False # Data incremental (task-agnostic/task-free) # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- SCENARIO CREATION n_classes = 10 task_scenario = SplitMNIST( nb_tasks, return_task_id=return_task_id, fixed_class_order=[i for i in range(n_classes)], ) # Make data incremental (one batch = one experience) scenario = data_incremental_benchmark(task_scenario, experience_size=batch_size) print( f"{scenario.n_experiences} batches in online data incremental setup.") # 6002 batches for SplitMNIST with batch size 10 # --------- # MODEL CREATION model = SimpleMLP(num_classes=args.featsize, hidden_size=400, hidden_layers=2, drop_rate=0) # choose some metrics and evaluation method logger = TextLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(experience=True, stream=True), loss_metrics(experience=False, stream=True), StreamForgetting(), loggers=[logger], benchmark=scenario, ) # CoPE PLUGIN cope = CoPEPlugin(mem_size=2000, alpha=0.99, p_size=args.featsize, n_classes=n_classes) # CREATE THE STRATEGY INSTANCE (NAIVE) WITH CoPE PLUGIN cl_strategy = Naive( model, torch.optim.SGD(model.parameters(), lr=0.01), cope.ppp_loss, # CoPE PPP-Loss train_mb_size=batch_size, train_epochs=epochs, eval_mb_size=100, device=device, plugins=[cope], evaluator=eval_plugin, ) # TRAINING LOOP print("Starting experiment...") results = [] cl_strategy.train(scenario.train_stream) print("Computing accuracy on the whole test set") results.append(cl_strategy.eval(scenario.test_stream))
def main(): args = parser.parse_args() args.cuda = args.cuda == 'yes' args.disable_pbar = args.disable_pbar == 'yes' args.stable_sgd = args.stable_sgd == 'yes' print(f"args={vars(args)}") device = torch.device("cuda:0" if torch.cuda.is_available() and args.cuda else "cpu") print(f'Using device: {device}') # unique identifier uid = uuid.uuid4().hex if args.uid is None else args.uid now = str(datetime.datetime.now().date()) + "_" + ':'.join(str(datetime.datetime.now().time()).split(':')[:-1]) runname = 'T={}_id={}'.format(now, uid) if not args.resume else args.resume # Paths setupname = [args.strategy, args.exp_name, args.model, args.scenario] parentdir = os.path.join(args.save_path, '_'.join(setupname)) results_path = Path(os.path.join(parentdir, runname)) results_path.mkdir(parents=True, exist_ok=True) tb_log_dir = os.path.join(results_path, 'tb_run') # Group all runs # Eval results eval_metric = 'Top1_Acc_Stream/eval_phase/test_stream' eval_results_dir = results_path / eval_metric.split('/')[0] eval_results_dir.mkdir(parents=True, exist_ok=True) eval_result_files = [] # To avg over seeds seeds = [args.seed] if args.seed is not None else list(range(args.n_seeds)) for seed in seeds: # initialize seeds print("STARTING SEED {}/{}".format(seed, len(seeds) - 1)) set_seed(seed) # create scenario if args.scenario == 'smnist': inputsize = 28 * 28 scenario = SplitMNIST(n_experiences=5, return_task_id=False, seed=seed, fixed_class_order=[i for i in range(10)]) elif args.scenario == 'CIFAR10': scenario = SplitCIFAR10(n_experiences=5, return_task_id=False, seed=seed, fixed_class_order=[i for i in range(10)]) inputsize = (3, 32, 32) elif args.scenario == 'miniimgnet': scenario = SplitMiniImageNet(args.dset_rootpath, n_experiences=20, return_task_id=False, seed=seed, fixed_class_order=[i for i in range(100)]) inputsize = (3, 84, 84) else: raise ValueError("Wrong scenario name.") print(f"Scenario = {args.scenario}") if args.model == 'simple_mlp': model = MyMLP(input_size=inputsize, hidden_size=args.hs) elif args.model == 'resnet18': if not args.stable_sgd: assert args.drop_prob == 0 model = ResNet18(inputsize, scenario.n_classes, drop_prob=args.drop_prob) criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=args.lr) # Paths eval_results_file = eval_results_dir / f'seed={seed}.csv' # LOGGING tb_logger = TensorboardLogger(tb_log_dir=tb_log_dir, tb_log_exp_name=f'seed={seed}.pt') # log to Tensorboard print_logger = TextLogger() if args.disable_pbar else InteractiveLogger() # print to stdout eval_logger = EvalTextLogger(metric_filter=eval_metric, file=open(eval_results_file, 'a')) eval_result_files.append(eval_results_file) # METRICS eval_plugin = EvaluationPlugin( accuracy_metrics(experience=True, stream=True), loss_metrics(minibatch=True, experience=True), ExperienceForgetting(), # Test only StreamConfusionMatrix(num_classes=scenario.n_classes, save_image=True), # LOG OTHER STATS # timing_metrics(epoch=True, experience=False), # cpu_usage_metrics(experience=True), # DiskUsageMonitor(), # MinibatchMaxRAM(), # GpuUsageMonitor(0), loggers=[print_logger, tb_logger, eval_logger]) plugins = None if args.strategy == 'replay': plugins = [RehRevPlugin(n_total_memories=args.mem_size, mode=args.replay_mode, # STEP-BACK aversion_steps=args.aversion_steps, aversion_lr=args.aversion_lr, stable_sgd=args.stable_sgd, # Stable SGD lr_decay=args.lr_decay, init_epochs=args.init_epochs # First task epochs )] # CREATE THE STRATEGY INSTANCE (NAIVE) strategy = Naive(model, optimizer, criterion, train_epochs=args.epochs, device=device, train_mb_size=args.bs, evaluator=eval_plugin, plugins=plugins ) # train on the selected scenario with the chosen strategy print('Starting experiment...') for experience in scenario.train_stream: if experience.current_experience == args.until_task: print("CUTTING OF TRAINING AT TASK ", experience.current_experience) break else: print("Start training on step ", experience.current_experience) strategy.train(experience) print("End training on step ", experience.current_experience) print('Computing accuracy on the test set') res = strategy.eval(scenario.test_stream[:args.until_task]) # Gathered by EvalLogger final_results_file = eval_results_dir / f'seed_summary.pt' stat_summarize(eval_result_files, final_results_file) print(f"[FILE:TB-RESULTS]: {tb_log_dir}") print(f"[FILE:FINAL-RESULTS]: {final_results_file}") print("FINISHED SCRIPT")
train_data_y=dataset[1], test_data_x=dataset[2], test_data_y=dataset[3], task_labels=[ 0 for key in task_order_list[task_order].keys() ], # shouldn't provide task ID for inference ) # log to Tensorboard tb_logger = TensorboardLogger( f"./tb_data/{cur_time}_simple_mlp_task_0_in_task_{task_order}/" ) # log to text file text_logger = TextLogger( open( f"./logs/{cur_time}_simple_mlp_task_0_in_task{task_order}.txt", "w+") ) # print to stdout interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(epoch=True, epoch_running=True), ExperienceForgetting(), cpu_usage_metrics(experience=True), StreamConfusionMatrix(num_classes=2, save_image=False), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True),
normalize, ]) test_transform = transforms.Compose([ transforms.Resize(224), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ]) root = ".." # log to Tensorboard tb_logger = TensorboardLogger(root) # log to text file text_logger = TextLogger(open(f"{root}/log.txt", "w")) # print to stdout interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(epoch=True, epoch_running=True), forgetting_metrics(experience=True, stream=True), cpu_usage_metrics(experience=True), confusion_matrix_metrics(num_classes=NUM_CLASSES[data_name], save_image=False, stream=True), disk_usage_metrics(minibatch=True, epoch=True,
test_datasets) scenario = scenario_custom_task_labels if (args.grow_classifier): print("using incremental classifier") model = DCNNNoVAEIncremental(num_classes=2) else: model = DCNNNoVAE(num_classes=5) # 5 because I happen to know this # log to Tensorboard path = args.tb_log_dir + "/" + ("".join(str( datetime.now()).split(" "))) + "_" + args.cl_strategy tb_logger = TensorboardLogger(tb_log_dir=path) # log to text file text_logger = TextLogger(open('log.txt', 'a')) # print to stdout interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=False, epoch=False, experience=True, stream=True), #loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), #timing_metrics(epoch=True), #cpu_usage_metrics(experience=True), #forgetting_metrics(experience=True, stream=True), #StreamConfusionMatrix(num_classes=5, save_image=False), StreamConfusionMatrix(save_image=False), #disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True) loggers=[interactive_logger, text_logger, tb_logger])
train_data_x=dataset[0], train_data_y=dataset[1], test_data_x=dataset[2], test_data_y=dataset[3], task_labels=[0 for key in perm.keys()], ) # Model Creation device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model = SimpleMLP(num_classes=2, input_size=70, hidden_size=100) # log to Tensorboard tb_logger = TensorboardLogger(f"./tb_data/{cur_time}-simpleMLP_Domain/") # log to text file text_logger = TextLogger(open(f"./logs/{cur_time}-simpleMLP_Domain.txt", "w+")) # print to stdout interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(epoch=True, epoch_running=True), ExperienceForgetting(), cpu_usage_metrics(experience=True), StreamConfusionMatrix(num_classes=2, save_image=False), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), loggers=[interactive_logger, text_logger, tb_logger], )
def main(args): # --- CONFIG device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # --------- # --- TRANSFORMATIONS train_transform = transforms.Compose([ RandomCrop(28, padding=4), ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) test_transform = transforms.Compose( [ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # --------- # --- SCENARIO CREATION mnist_train = MNIST('./data/mnist', train=True, download=True, transform=train_transform) mnist_test = MNIST('./data/mnist', train=False, download=True, transform=test_transform) scenario = nc_scenario(mnist_train, mnist_test, 5, task_labels=False, seed=1234) # --------- # MODEL CREATION model = SimpleMLP(num_classes=scenario.n_classes) # DEFINE THE EVALUATION PLUGIN AND LOGGER # The evaluation plugin manages the metrics computation. # It takes as argument a list of metrics and a list of loggers. # The evaluation plugin calls the loggers to serialize the metrics # and save them in persistent memory or print them in the standard output. # log to text file text_logger = TextLogger(open('log.txt', 'a')) # print to stdout interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin(accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), cpu_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, experience=True, stream=True), ExperienceForgetting(), loggers=[interactive_logger, text_logger]) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=500, train_epochs=1, eval_mb_size=100, device=device, evaluator=eval_plugin) # TRAINING LOOP print('Starting experiment...') results = [] for experience in scenario.train_stream: print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) # train returns a list of dictionaries (one for each experience). Each # dictionary stores the last value of each metric curve emitted # during training. res = cl_strategy.train(experience) print('Training completed') print('Computing accuracy on the whole test set') # test also returns a dictionary results.append(cl_strategy.eval(scenario.test_stream)) print(f"Test metrics:\n{results}") # All the metric curves (x,y values) are stored inside the evaluator # (can be disabled). You can use this dictionary to manipulate the # metrics without avalanche. all_metrics = cl_strategy.evaluator.all_metrics print(f"Stored metrics: {list(all_metrics.keys())}") mname = 'Top1_Acc_Task/Task000' print(f"{mname}: {cl_strategy.evaluator.all_metrics[mname]}")
def test_text_logger(self): logp = TextLogger() self._test_logger(logp)
generic_scenario = tensor_scenario( train_data_x=dataset[0], train_data_y=dataset[1], test_data_x=dataset[2], test_data_y=dataset[3], task_labels=[0 for key in task_order_list[task_order].keys() ], # shouldn't provide task ID for inference ) # log to Tensorboard tb_logger = TensorboardLogger( f"./tb_data/{cur_time}_CNN2D_ClassInc_0_in_task{task_order+1}/") # log to text file text_logger = TextLogger( open(f"./logs/{cur_time}_CNN2D_ClassInc_0_in_task{task_order+1}.txt", "w+")) # print to stdout interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(epoch=True, epoch_running=True), ExperienceForgetting(), cpu_usage_metrics(experience=True), StreamConfusionMatrix(num_classes=2, save_image=False),
generic_scenario = tensor_scenario( train_data_x=dataset[0], train_data_y=dataset[1], test_data_x=dataset[2], test_data_y=dataset[3], task_labels=[ 0 for key in task_order_list[task_order].keys() ], # shouldn't provide task ID for inference ) # log to Tensorboard tb_logger = TensorboardLogger(f"./tb_data/{cur_time}_CNN2D_0task_{task_order+1}/") # log to text file text_logger = TextLogger( open(f"./logs/{cur_time}_CNN2D_0task_{task_order+1}.txt", "w+") ) # print to stdout interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), timing_metrics(epoch=True, epoch_running=True), ExperienceForgetting(), cpu_usage_metrics(experience=True), StreamConfusionMatrix(num_classes=2, save_image=False), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), loggers=[interactive_logger, text_logger, tb_logger], )
def setUpClass(cls) -> None: torch.manual_seed(0) np.random.seed(0) random.seed(0) n_samples_per_class = 100 dataset = make_classification( n_samples=6 * n_samples_per_class, n_classes=6, n_features=4, n_informative=4, n_redundant=0, ) X = torch.from_numpy(dataset[0]).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split(X, y, train_size=0.5, shuffle=True, stratify=y) tr_d = TensorDataset(train_X, train_y) ts_d = TensorDataset(test_X, test_y) benchmark = nc_benchmark( train_dataset=tr_d, test_dataset=ts_d, n_experiences=3, task_labels=True, shuffle=False, seed=0, ) model = SimpleMLP(input_size=4, num_classes=benchmark.n_classes) f = open("log.txt", "w") text_logger = TextLogger(f) eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, trained_experience=True, ), loss_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), forgetting_metrics(experience=True, stream=True), confusion_matrix_metrics(num_classes=6, save_image=False, normalize="all", stream=True), bwt_metrics(experience=True, stream=True), forward_transfer_metrics(experience=True, stream=True), cpu_usage_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), timing_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), ram_usage_metrics( every=0.5, minibatch=True, epoch=True, experience=True, stream=True, ), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[text_logger], collect_all=True, ) # collect all metrics (set to True by default) cl_strategy = BaseStrategy( model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=10, train_epochs=2, eval_mb_size=10, device=DEVICE, evaluator=eval_plugin, eval_every=1, ) for i, experience in enumerate(benchmark.train_stream): cl_strategy.train(experience, eval_streams=[benchmark.test_stream], shuffle=False) cl_strategy.eval(benchmark.test_stream) cls.all_metrics = cl_strategy.evaluator.get_all_metrics() f.close() # Set the environment variable UPDATE_METRICS to True to update # the pickle file with target values. # Make sure the old tests were passing for all unchanged metrics if UPDATE_METRICS: with open( os.path.join( pathlib.Path(__file__).parent.absolute(), "target_metrics", "mt.pickle", ), "wb", ) as f: pickle.dump(dict(cls.all_metrics), f, protocol=4) with open( os.path.join( pathlib.Path(__file__).parent.absolute(), "target_metrics", "mt.pickle", ), "rb", ) as f: cls.ref = pickle.load(f)