def test_cumulative(self): model = self.get_model(fast_test=self.fast_test) optimizer = SGD(model.parameters(), lr=1e-3) criterion = CrossEntropyLoss() # SIT scenario my_nc_scenario = self.load_scenario(fast_test=self.fast_test) strategy = Cumulative(model, optimizer, criterion, train_mb_size=64, device=self.device, eval_mb_size=50, train_epochs=2) self.run_strategy(my_nc_scenario, strategy) # MT scenario strategy = Cumulative(model, optimizer, criterion, train_mb_size=64, device=self.device, eval_mb_size=50, train_epochs=2) scenario = self.load_scenario(fast_test=self.fast_test, use_task_labels=True) self.run_strategy(scenario, strategy)
def test_cumulative(self): # SIT scenario model, optimizer, criterion, my_nc_benchmark = self.init_sit() strategy = Cumulative( model, optimizer, criterion, train_mb_size=64, device=self.device, eval_mb_size=50, train_epochs=2, ) self.run_strategy(my_nc_benchmark, strategy) # MT scenario strategy = Cumulative( model, optimizer, criterion, train_mb_size=64, device=self.device, eval_mb_size=50, train_epochs=2, ) benchmark = self.load_benchmark(use_task_labels=True) self.run_strategy(benchmark, strategy)
def test_early_stop(self): class EarlyStopP(StrategyPlugin): def after_training_iteration(self, strategy: 'BaseStrategy', **kwargs): if strategy.mb_it == 10: strategy.stop_training() model = SimpleMLP(input_size=6, hidden_size=100) criterion = CrossEntropyLoss() optimizer = SGD(model.parameters(), lr=1) strategy = Cumulative(model, optimizer, criterion, train_mb_size=1, device=get_device(), eval_mb_size=512, train_epochs=1, evaluator=None, plugins=[EarlyStopP()]) scenario = get_fast_scenario() for train_batch_info in scenario.train_stream: strategy.train(train_batch_info) assert strategy.mb_it == 11
def test_multihead_cumulative(self): # check that multi-head reaches high enough accuracy. # Ensure nothing weird is happening with the multiple heads. model = MHTestMLP(input_size=6, hidden_size=100) criterion = CrossEntropyLoss() optimizer = SGD(model.parameters(), lr=1) main_metric = StreamAccuracy() exp_acc = ExperienceAccuracy() evalp = EvaluationPlugin(main_metric, exp_acc, loggers=None) strategy = Cumulative(model, optimizer, criterion, train_mb_size=32, device=get_device(), eval_mb_size=512, train_epochs=1, evaluator=evalp) scenario = get_fast_scenario(use_task_labels=True) for train_batch_info in scenario.train_stream: strategy.train(train_batch_info) strategy.eval(scenario.train_stream[:]) print("TRAIN STREAM ACC: ", main_metric.result()) assert sum(main_metric.result().values()) / \ float(len(main_metric.result().keys())) > 0.7