def test_early_stop(self): class EarlyStopP(StrategyPlugin): def after_training_iteration(self, strategy: 'BaseStrategy', **kwargs): if strategy.mb_it == 10: strategy.stop_training() model = SimpleMLP(input_size=6, hidden_size=100) criterion = CrossEntropyLoss() optimizer = SGD(model.parameters(), lr=1) strategy = Cumulative(model, optimizer, criterion, train_mb_size=1, device=get_device(), eval_mb_size=512, train_epochs=1, evaluator=None, plugins=[EarlyStopP()]) scenario = get_fast_scenario() for train_batch_info in scenario.train_stream: strategy.train(train_batch_info) assert strategy.mb_it == 11
def test_multihead_cumulative(self): # check that multi-head reaches high enough accuracy. # Ensure nothing weird is happening with the multiple heads. model = MHTestMLP(input_size=6, hidden_size=100) criterion = CrossEntropyLoss() optimizer = SGD(model.parameters(), lr=1) main_metric = StreamAccuracy() exp_acc = ExperienceAccuracy() evalp = EvaluationPlugin(main_metric, exp_acc, loggers=None) strategy = Cumulative(model, optimizer, criterion, train_mb_size=32, device=get_device(), eval_mb_size=512, train_epochs=1, evaluator=evalp) scenario = get_fast_scenario(use_task_labels=True) for train_batch_info in scenario.train_stream: strategy.train(train_batch_info) strategy.eval(scenario.train_stream[:]) print("TRAIN STREAM ACC: ", main_metric.result()) assert sum(main_metric.result().values()) / \ float(len(main_metric.result().keys())) > 0.7
def _test_replay_balanced_memory(self, storage_policy, mem_size): scenario = get_fast_scenario(use_task_labels=True) model = SimpleMLP(input_size=6, hidden_size=10) replayPlugin = ReplayPlugin(mem_size=mem_size, storage_policy=storage_policy) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001), CrossEntropyLoss(), train_mb_size=32, train_epochs=1, eval_mb_size=100, plugins=[replayPlugin]) n_seen_data = 0 for step in scenario.train_stream: n_seen_data += len(step.dataset) mem_fill = min(mem_size, n_seen_data) cl_strategy.train(step) ext_mem = replayPlugin.ext_mem lengths = [] for task_id in ext_mem.keys(): lengths.append(len(ext_mem[task_id])) self.assertEqual(sum(lengths), mem_fill) # Always fully filled
def load_scenario(self, use_task_labels=False): """ Returns a NC Scenario from a fake dataset of 10 classes, 5 experiences, 2 classes per experience. :param fast_test: if True loads fake data, MNIST otherwise. """ return get_fast_scenario(use_task_labels=use_task_labels)
def test_periodic_eval(self): model = SimpleMLP(input_size=6, hidden_size=10) scenario = get_fast_scenario() optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() curve_key = 'Top1_Acc_Stream/eval_phase/train_stream/Task000' ################### # Case #1: No eval ################### # we use stream acc. because it emits a single value # for each eval loop. acc = StreamAccuracy() strategy = Naive(model, optimizer, criterion, train_epochs=2, eval_every=-1, evaluator=EvaluationPlugin(acc)) strategy.train(scenario.train_stream[0]) # eval is not called in this case assert len(strategy.evaluator.get_all_metrics()) == 0 ################### # Case #2: Eval at the end only ################### acc = StreamAccuracy() strategy = Naive(model, optimizer, criterion, train_epochs=2, eval_every=0, evaluator=EvaluationPlugin(acc)) strategy.train(scenario.train_stream[0]) # eval is called once at the end of the training loop curve = strategy.evaluator.get_all_metrics()[curve_key][1] assert len(curve) == 1 ################### # Case #3: Eval after every epoch ################### acc = StreamAccuracy() strategy = Naive(model, optimizer, criterion, train_epochs=2, eval_every=1, evaluator=EvaluationPlugin(acc)) strategy.train(scenario.train_stream[0]) # eval is called after every epoch + the end of the training loop curve = strategy.evaluator.get_all_metrics()[curve_key][1] assert len(curve) == 3
def test_multihead_head_creation(self): # Check if the optimizer is updated correctly # when heads are created and updated. model = MTSimpleMLP(input_size=6, hidden_size=10) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() scenario = get_fast_scenario(use_task_labels=True, shuffle=False) strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu') strategy.evaluator.loggers = [TextLogger(sys.stdout)] print("Current Classes: ", scenario.train_stream[4].classes_in_this_experience) print("Current Classes: ", scenario.train_stream[0].classes_in_this_experience) # head creation strategy.train(scenario.train_stream[0]) w_ptr = model.classifier.classifiers['0'].classifier.weight.data_ptr() b_ptr = model.classifier.classifiers['0'].classifier.bias.data_ptr() opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group['params'] ] assert w_ptr in opt_params_ptrs assert b_ptr in opt_params_ptrs # head update strategy.train(scenario.train_stream[4]) w_ptr_t0 = model.classifier.classifiers[ '0'].classifier.weight.data_ptr() b_ptr_t0 = model.classifier.classifiers['0'].classifier.bias.data_ptr() w_ptr_new = model.classifier.classifiers[ '4'].classifier.weight.data_ptr() b_ptr_new = model.classifier.classifiers['4'].classifier.bias.data_ptr( ) opt_params_ptrs = [ w.data_ptr() for group in optimizer.param_groups for w in group['params'] ] assert w_ptr not in opt_params_ptrs # head0 has been updated assert b_ptr not in opt_params_ptrs # head0 has been updated assert w_ptr_t0 in opt_params_ptrs assert b_ptr_t0 in opt_params_ptrs assert w_ptr_new in opt_params_ptrs assert b_ptr_new in opt_params_ptrs
def assert_balancing(self, policy): ext_mem = policy.ext_mem scenario = get_fast_scenario(use_task_labels=True) replay = ReplayPlugin(mem_size=100, storage_policy=policy) model = SimpleMLP(num_classes=scenario.n_classes) # CREATE THE STRATEGY INSTANCE (NAIVE) cl_strategy = Naive(model, SGD(model.parameters(), lr=0.001), CrossEntropyLoss(), train_mb_size=100, train_epochs=0, eval_mb_size=100, plugins=[replay], evaluator=None) for exp in scenario.train_stream: cl_strategy.train(exp) print(list(ext_mem.keys()), [len(el) for el in ext_mem.values()]) # buffer size should equal self.mem_size if data is large enough len_tot = sum([len(el) for el in ext_mem.values()]) assert len_tot == policy.mem_size
def test_multihead_head_selection(self): # Check if the optimizer is updated correctly # when heads are created and updated. model = MultiHeadClassifier(in_features=6) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() scenario = get_fast_scenario(use_task_labels=True, shuffle=False) strategy = Naive(model, optimizer, criterion, train_mb_size=100, train_epochs=1, eval_mb_size=100, device='cpu') strategy.evaluator.loggers = [TextLogger(sys.stdout)] # initialize head strategy.train(scenario.train_stream[0]) strategy.train(scenario.train_stream[4]) # create models with fixed head model_t0 = model.classifiers['0'] model_t4 = model.classifiers['4'] # check head task0 for x, y, t in DataLoader(scenario.train_stream[0].dataset): y_mh = model(x, t) y_t = model_t0(x) assert ((y_mh - y_t)**2).sum() < 1.e-7 break # check head task4 for x, y, t in DataLoader(scenario.train_stream[4].dataset): y_mh = model(x, t) y_t = model_t4(x) assert ((y_mh - y_t)**2).sum() < 1.e-7 break
def test_pnn(self): # check that pnn reaches high enough accuracy. # Ensure nothing weird is happening with the multiple heads. main_metric = StreamAccuracy() exp_acc = ExperienceAccuracy() evalp = EvaluationPlugin(main_metric, exp_acc, loggers=None) strategy = PNNStrategy(1, 6, 50, 0.1, train_mb_size=32, device=get_device(), eval_mb_size=512, train_epochs=1, evaluator=evalp) scenario = get_fast_scenario(use_task_labels=True) for train_batch_info in scenario.train_stream: strategy.train(train_batch_info) strategy.eval(scenario.train_stream[:]) print("TRAIN STREAM ACC: ", main_metric.result()) assert main_metric.result() > 0.5
def setUp(self): common_setups() self.scenario = get_fast_scenario(use_task_labels=False, shuffle=False)