def test_last_batch_baseline_model(): """ BUG: Baseline method is doing something weird at the last batch, and I dont know quite why. """ n_samples = 110 batch_size = 20 # Note: the y's here are different. dataset = TensorDataset( torch.arange(n_samples).reshape([n_samples, 1, 1, 1]) * torch.ones([n_samples, 3, 32, 32]), torch.zeros(n_samples, dtype=int), ) pretend_to_be_active = False env = PassiveEnvironment( dataset, batch_size=batch_size, n_classes=n_samples, pretend_to_be_active=pretend_to_be_active, ) env = TypedObjectsWrapper(env, observations_type=Observations, actions_type=Actions, rewards_type=Rewards) env = MeasureSLPerformanceWrapper(env, first_epoch_only=True) setting = ClassIncrementalSetting() setting.train_env = env model = BaselineModel(setting=setting, hparams=BaselineModel.HParams(), config=Config(debug=True)) for i, (obs, rew) in enumerate(env): # assert rew is None # if i != 5: # assert obs.batch_size == 20, i # else: # assert obs.batch_size == 10, i # actions = Actions(y_pred=torch.arange(i * 20 , (i+1) * 20)[:obs.batch_size]) # rewards = env.send(actions) # assert (rewards.y == torch.arange(i * 20 , (i+1) * 20)[:obs.batch_size]).all() obs = dataclasses.replace(obs, task_labels=torch.ones([obs.x.shape[0]], device=obs.x.device)) assert rew is None stuff = model.training_step((obs, rew), batch_idx=i) print(stuff) perf = env.get_average_online_performance() assert perf.n_samples == 110
def demo_simple(): """ Simple demo: Creating and applying a Method onto a Setting. """ from sequoia.settings import DomainIncrementalSetting ## 1. Creating the setting: setting = DomainIncrementalSetting(dataset="fashionmnist", batch_size=32) ## 2. Creating the Method method = DemoMethod() # (Optional): You can also create a Config, which holds other fields like # `log_dir`, `debug`, `device`, etc. which aren't specific to either the # Setting or the Method. config = Config(debug=True, render=False) ## 3. Applying the method to the setting: (optionally passing a Config to # use for that run) results = setting.apply(method, config=config) print(results.summary()) print(f"objective: {results.objective}")
def baseline_demo_simple(): config = Config() method = BaselineMethod(config=config, max_epochs=1) ## Create *any* Setting from the tree, for example: ## Supervised Learning Setting: # setting = TaskIncrementalSLSetting( # dataset="cifar10", # nb_tasks=2, # ) # Reinforcement Learning Setting: setting = TaskIncrementalRLSetting( dataset="cartpole", max_steps=4000, nb_tasks=2, ) results = setting.apply(method, config=config) print(results.summary()) return results
def prepare_data(self, *args, **kwargs) -> None: # We don't really download anything atm. if self.config is None: self.config = Config() super().prepare_data(*args, **kwargs)
y_pred = action_space.sample() return self.target_setting.Actions(y_pred) if __name__ == "__main__": from sequoia.common import Config from sequoia.settings import ClassIncrementalSetting # Create the Method: # - Manually: method = DummyMethod() # NOTE: This Setting is very similar to the one used for the SL track of the # competition. from sequoia.client import SettingProxy setting = SettingProxy(ClassIncrementalSetting, "sl_track") # setting = SettingProxy(ClassIncrementalSetting, # dataset="synbols", # nb_tasks=12, # known_task_boundaries_at_test_time=False, # monitor_training_performance=True, # batch_size=32, # num_workers=4, # ) # NOTE: can also use pass a `Config` object to `setting.apply`. This object has some # configuration options like device, data_dir, etc. results = setting.apply(method, config=Config(data_dir="data")) print(results.summary())
def __init__( self, hparams: BaselineModel.HParams = None, config: Config = None, trainer_options: TrainerConfig = None, **kwargs, ): """ Creates a new BaselineMethod, using the provided configuration options. Parameters ---------- hparams : BaselineModel.HParams, optional Hyper-parameters of the BaselineModel used by this Method. Defaults to None. config : Config, optional Configuration dataclass with options like log_dir, device, etc. Defaults to None. trainer_options : TrainerConfig, optional Dataclass which holds all the options for creating the `pl.Trainer` which will be used for training. Defaults to None. **kwargs : If any of the above arguments are left as `None`, then they will be created using any appropriate value from `kwargs`, if present. ## Examples: ``` method = BaselineMethod(hparams=BaselineModel.HParams(learning_rate=0.01)) method = BaselineMethod(learning_rate=0.01) # Same as above method = BaselineMethod(config=Config(debug=True)) method = BaselineMethod(debug=True) # Same as above method = BaselineMethod(hparams=BaselineModel.HParams(learning_rate=0.01), config=Config(debug=True)) method = BaselineMethod(learning_rate=0.01, debug=True) # Same as above ``` """ # TODO: When creating a Method from a script, like `BaselineMethod()`, # should we expect the hparams to be passed? Should we create them from # the **kwargs? Should we parse them from the command-line? # Option 2: Try to use the keyword arguments to create the hparams, # config and trainer options. if kwargs: logger.info( f"using keyword arguments {kwargs} to populate the corresponding " f"values in the hparams, config and trainer_options.") self.hparams = hparams or BaselineModel.HParams.from_dict( kwargs, drop_extra_fields=True) self.config = config or Config.from_dict(kwargs, drop_extra_fields=True) self.trainer_options = trainer_options or TrainerConfig.from_dict( kwargs, drop_extra_fields=True) elif self._argv: # Since the method was parsed from the command-line, parse those as # well from the argv that were used to create the Method. # Option 3: Parse them from the command-line. # assert not kwargs, "Don't pass any extra kwargs to the constructor!" self.hparams = hparams or BaselineModel.HParams.from_args( self._argv, strict=False) self.config = config or Config.from_args(self._argv, strict=False) self.trainer_options = trainer_options or TrainerConfig.from_args( self._argv, strict=False) else: # Option 1: Use the default values: self.hparams = hparams or BaselineModel.HParams() self.config = config or Config() self.trainer_options = trainer_options or TrainerConfig() assert self.hparams assert self.config assert self.trainer_options if self.config.debug: # Disable wandb logging if debug is True. self.trainer_options.no_wandb = True # The model and Trainer objects will be created in `self.configure`. # NOTE: This right here doesn't create the fields, it just gives some # type information for static type checking. self.trainer: Trainer self.model: BaselineModel self.additional_train_wrappers: List[Callable] = [] self.additional_valid_wrappers: List[Callable] = [] self.setting: Setting
def configure(self, setting: SettingType) -> None: """Configures the method for the given Setting. Concretely, this creates the model and Trainer objects which will be used to train and test a model for the given `setting`. Args: setting (SettingType): The setting the method will be evaluated on. TODO: For the Challenge, this should be some kind of read-only proxy to the actual Setting. """ # Note: this here is temporary, just tinkering with wandb atm. method_name: str = self.get_name() # Set the default batch size to use, depending on the kind of Setting. if self.hparams.batch_size is None: if isinstance(setting, ActiveSetting): # Default batch size of 1 in RL self.hparams.batch_size = 1 elif isinstance(setting, PassiveSetting): self.hparams.batch_size = 32 else: warnings.warn( UserWarning( f"Dont know what batch size to use by default for setting " f"{setting}, will try 16.")) self.hparams.batch_size = 16 # Set the batch size on the setting. setting.batch_size = self.hparams.batch_size # TODO: Should we set the 'config' on the setting from here? if setting.config and setting.config == self.config: pass elif self.config != Config(): assert ( setting.config is None or setting.config == Config() ), "method.config has been modified, and so has setting.config!" setting.config = self.config elif setting.config: assert (setting.config != Config()), "Weird, both configs have default values.." self.config = setting.config setting_name: str = setting.get_name() dataset = setting.dataset if isinstance(setting, IncrementalSetting): if self.hparams.multihead is None: # Use a multi-head model by default if the task labels are # available at both train and test time. if setting.task_labels_at_test_time: assert setting.task_labels_at_train_time self.hparams.multihead = setting.task_labels_at_test_time if isinstance(setting, ContinualRLSetting): setting.add_done_to_observations = True if not setting.observe_state_directly: if self.hparams.encoder is None: self.hparams.encoder = "simple_convnet" # TODO: Add 'proper' transforms for cartpole, specifically? from sequoia.common.transforms import Transforms setting.train_transforms.append(Transforms.resize_64x64) setting.val_transforms.append(Transforms.resize_64x64) setting.test_transforms.append(Transforms.resize_64x64) # Configure the baseline specifically for an RL setting. # TODO: Select which output head to use from the command-line? # Limit the number of epochs so we never iterate on a closed env. # TODO: Would multiple "epochs" be possible? if setting.max_steps is not None: self.trainer_options.max_epochs = 1 self.trainer_options.limit_train_batches = setting.max_steps // ( setting.batch_size or 1) self.trainer_options.limit_val_batches = min( setting.max_steps // (setting.batch_size or 1), 1000) # TODO: Test batch size is limited to 1 for now. # NOTE: This isn't used, since we don't call `trainer.test()`. self.trainer_options.limit_test_batches = setting.max_steps self.model = self.create_model(setting) assert self.hparams is self.model.hp # The PolicyHead actually does its own backward pass, so we disable # automatic optimization when using it. from .models.output_heads import PolicyHead if isinstance(self.model.output_head, PolicyHead): # Doing the backward pass manually, since there might not be a loss # at each step. self.trainer_options.automatic_optimization = False self.trainer = self.create_trainer(setting) self.setting = setting
# setting = DomainIncrementalSetting( # dataset="mnist", nb_tasks=5, monitor_training_performance=True # ) # - "Medium": Class-Incremental MNIST Setting, useful for quick debugging: # setting = ClassIncrementalSetting( # dataset="mnist", # nb_tasks=5, # monitor_training_performance=True, # known_task_boundaries_at_test_time=False, # batch_size=32, # num_workes=4, # ) # - "HARD": Class-Incremental Synbols, more challenging. # NOTE: This Setting is very similar to the one used for the SL track of the # competition. setting = ClassIncrementalSetting( dataset="synbols", nb_tasks=12, known_task_boundaries_at_test_time=False, monitor_training_performance=True, batch_size=32, num_workers=4, ) # Run the experiment: results = setting.apply(method, config=Config(debug=True, data_dir="./data")) print(results.summary())
# from sequoia.settings.sl.class_incremental.domain_incremental import DomainIncrementalSLSetting # setting = DomainIncrementalSLSetting( # dataset="mnist", nb_tasks=5, monitor_training_performance=True # ) # - "Medium": Class-Incremental MNIST Setting, useful for quick debugging: # setting = ClassIncrementalSetting( # dataset="mnist", # nb_tasks=5, # monitor_training_performance=True, # known_task_boundaries_at_test_time=False, # batch_size=32, # num_workes=4, # ) # - "HARD": Class-Incremental Synbols, more challenging. # NOTE: This Setting is very similar to the one used for the SL track of the # competition. setting = ClassIncrementalSetting( dataset="synbols", nb_tasks=12, known_task_boundaries_at_test_time=False, monitor_training_performance=True, batch_size=32, num_workers=4, ) # Run the experiment: results = setting.apply(method, config=Config(debug=True, data_dir="./data")) print(results.summary())