def test_last_batch_baseline_model():
""" BUG: Baseline method is doing something weird at the last batch, and I dont know quite why.
"""
n_samples = 110
batch_size = 20
# Note: the y's here are different.
dataset = TensorDataset(
torch.arange(n_samples).reshape([n_samples, 1, 1, 1]) *
torch.ones([n_samples, 3, 32, 32]),
torch.zeros(n_samples, dtype=int),
)
pretend_to_be_active = False
env = PassiveEnvironment(
dataset,
batch_size=batch_size,
n_classes=n_samples,
pretend_to_be_active=pretend_to_be_active,
)
env = TypedObjectsWrapper(env,
observations_type=Observations,
actions_type=Actions,
rewards_type=Rewards)
env = MeasureSLPerformanceWrapper(env, first_epoch_only=True)
setting = ClassIncrementalSetting()
setting.train_env = env
model = BaselineModel(setting=setting,
hparams=BaselineModel.HParams(),
config=Config(debug=True))
for i, (obs, rew) in enumerate(env):
# assert rew is None
# if i != 5:
# assert obs.batch_size == 20, i
# else:
# assert obs.batch_size == 10, i
# actions = Actions(y_pred=torch.arange(i * 20 , (i+1) * 20)[:obs.batch_size])
# rewards = env.send(actions)
# assert (rewards.y == torch.arange(i * 20 , (i+1) * 20)[:obs.batch_size]).all()
obs = dataclasses.replace(obs,
task_labels=torch.ones([obs.x.shape[0]],
device=obs.x.device))
assert rew is None
stuff = model.training_step((obs, rew), batch_idx=i)
print(stuff)
perf = env.get_average_online_performance()
assert perf.n_samples == 110
def demo_simple():
""" Simple demo: Creating and applying a Method onto a Setting. """
from sequoia.settings import DomainIncrementalSetting
## 1. Creating the setting:
setting = DomainIncrementalSetting(dataset="fashionmnist", batch_size=32)
## 2. Creating the Method
method = DemoMethod()
# (Optional): You can also create a Config, which holds other fields like
# `log_dir`, `debug`, `device`, etc. which aren't specific to either the
# Setting or the Method.
config = Config(debug=True, render=False)
## 3. Applying the method to the setting: (optionally passing a Config to
# use for that run)
results = setting.apply(method, config=config)
print(results.summary())
print(f"objective: {results.objective}")
def baseline_demo_simple():
config = Config()
method = BaselineMethod(config=config, max_epochs=1)
## Create *any* Setting from the tree, for example:
## Supervised Learning Setting:
# setting = TaskIncrementalSLSetting(
# dataset="cifar10",
# nb_tasks=2,
# )
# Reinforcement Learning Setting:
setting = TaskIncrementalRLSetting(
dataset="cartpole",
max_steps=4000,
nb_tasks=2,
)
results = setting.apply(method, config=config)
print(results.summary())
return results
def prepare_data(self, *args, **kwargs) -> None:
# We don't really download anything atm.
if self.config is None:
self.config = Config()
super().prepare_data(*args, **kwargs)
y_pred = action_space.sample()
return self.target_setting.Actions(y_pred)
if __name__ == "__main__":
from sequoia.common import Config
from sequoia.settings import ClassIncrementalSetting
# Create the Method:
# - Manually:
method = DummyMethod()
# NOTE: This Setting is very similar to the one used for the SL track of the
# competition.
from sequoia.client import SettingProxy
setting = SettingProxy(ClassIncrementalSetting, "sl_track")
# setting = SettingProxy(ClassIncrementalSetting,
# dataset="synbols",
# nb_tasks=12,
# known_task_boundaries_at_test_time=False,
# monitor_training_performance=True,
# batch_size=32,
# num_workers=4,
# )
# NOTE: can also use pass a `Config` object to `setting.apply`. This object has some
# configuration options like device, data_dir, etc.
results = setting.apply(method, config=Config(data_dir="data"))
print(results.summary())
def __init__(
self,
hparams: BaselineModel.HParams = None,
config: Config = None,
trainer_options: TrainerConfig = None,
**kwargs,
):
""" Creates a new BaselineMethod, using the provided configuration options.
Parameters
----------
hparams : BaselineModel.HParams, optional
Hyper-parameters of the BaselineModel used by this Method. Defaults to None.
config : Config, optional
Configuration dataclass with options like log_dir, device, etc. Defaults to
None.
trainer_options : TrainerConfig, optional
Dataclass which holds all the options for creating the `pl.Trainer` which
will be used for training. Defaults to None.
**kwargs :
If any of the above arguments are left as `None`, then they will be created
using any appropriate value from `kwargs`, if present.
## Examples:
```
method = BaselineMethod(hparams=BaselineModel.HParams(learning_rate=0.01))
method = BaselineMethod(learning_rate=0.01) # Same as above
method = BaselineMethod(config=Config(debug=True))
method = BaselineMethod(debug=True) # Same as above
method = BaselineMethod(hparams=BaselineModel.HParams(learning_rate=0.01),
config=Config(debug=True))
method = BaselineMethod(learning_rate=0.01, debug=True) # Same as above
```
"""
# TODO: When creating a Method from a script, like `BaselineMethod()`,
# should we expect the hparams to be passed? Should we create them from
# the **kwargs? Should we parse them from the command-line?
# Option 2: Try to use the keyword arguments to create the hparams,
# config and trainer options.
if kwargs:
logger.info(
f"using keyword arguments {kwargs} to populate the corresponding "
f"values in the hparams, config and trainer_options.")
self.hparams = hparams or BaselineModel.HParams.from_dict(
kwargs, drop_extra_fields=True)
self.config = config or Config.from_dict(kwargs,
drop_extra_fields=True)
self.trainer_options = trainer_options or TrainerConfig.from_dict(
kwargs, drop_extra_fields=True)
elif self._argv:
# Since the method was parsed from the command-line, parse those as
# well from the argv that were used to create the Method.
# Option 3: Parse them from the command-line.
# assert not kwargs, "Don't pass any extra kwargs to the constructor!"
self.hparams = hparams or BaselineModel.HParams.from_args(
self._argv, strict=False)
self.config = config or Config.from_args(self._argv, strict=False)
self.trainer_options = trainer_options or TrainerConfig.from_args(
self._argv, strict=False)
else:
# Option 1: Use the default values:
self.hparams = hparams or BaselineModel.HParams()
self.config = config or Config()
self.trainer_options = trainer_options or TrainerConfig()
assert self.hparams
assert self.config
assert self.trainer_options
if self.config.debug:
# Disable wandb logging if debug is True.
self.trainer_options.no_wandb = True
# The model and Trainer objects will be created in `self.configure`.
# NOTE: This right here doesn't create the fields, it just gives some
# type information for static type checking.
self.trainer: Trainer
self.model: BaselineModel
self.additional_train_wrappers: List[Callable] = []
self.additional_valid_wrappers: List[Callable] = []
self.setting: Setting
def configure(self, setting: SettingType) -> None:
"""Configures the method for the given Setting.
Concretely, this creates the model and Trainer objects which will be
used to train and test a model for the given `setting`.
Args:
setting (SettingType): The setting the method will be evaluated on.
TODO: For the Challenge, this should be some kind of read-only proxy to the
actual Setting.
"""
# Note: this here is temporary, just tinkering with wandb atm.
method_name: str = self.get_name()
# Set the default batch size to use, depending on the kind of Setting.
if self.hparams.batch_size is None:
if isinstance(setting, ActiveSetting):
# Default batch size of 1 in RL
self.hparams.batch_size = 1
elif isinstance(setting, PassiveSetting):
self.hparams.batch_size = 32
else:
warnings.warn(
UserWarning(
f"Dont know what batch size to use by default for setting "
f"{setting}, will try 16."))
self.hparams.batch_size = 16
# Set the batch size on the setting.
setting.batch_size = self.hparams.batch_size
# TODO: Should we set the 'config' on the setting from here?
if setting.config and setting.config == self.config:
pass
elif self.config != Config():
assert (
setting.config is None or setting.config == Config()
), "method.config has been modified, and so has setting.config!"
setting.config = self.config
elif setting.config:
assert (setting.config !=
Config()), "Weird, both configs have default values.."
self.config = setting.config
setting_name: str = setting.get_name()
dataset = setting.dataset
if isinstance(setting, IncrementalSetting):
if self.hparams.multihead is None:
# Use a multi-head model by default if the task labels are
# available at both train and test time.
if setting.task_labels_at_test_time:
assert setting.task_labels_at_train_time
self.hparams.multihead = setting.task_labels_at_test_time
if isinstance(setting, ContinualRLSetting):
setting.add_done_to_observations = True
if not setting.observe_state_directly:
if self.hparams.encoder is None:
self.hparams.encoder = "simple_convnet"
# TODO: Add 'proper' transforms for cartpole, specifically?
from sequoia.common.transforms import Transforms
setting.train_transforms.append(Transforms.resize_64x64)
setting.val_transforms.append(Transforms.resize_64x64)
setting.test_transforms.append(Transforms.resize_64x64)
# Configure the baseline specifically for an RL setting.
# TODO: Select which output head to use from the command-line?
# Limit the number of epochs so we never iterate on a closed env.
# TODO: Would multiple "epochs" be possible?
if setting.max_steps is not None:
self.trainer_options.max_epochs = 1
self.trainer_options.limit_train_batches = setting.max_steps // (
setting.batch_size or 1)
self.trainer_options.limit_val_batches = min(
setting.max_steps // (setting.batch_size or 1), 1000)
# TODO: Test batch size is limited to 1 for now.
# NOTE: This isn't used, since we don't call `trainer.test()`.
self.trainer_options.limit_test_batches = setting.max_steps
self.model = self.create_model(setting)
assert self.hparams is self.model.hp
# The PolicyHead actually does its own backward pass, so we disable
# automatic optimization when using it.
from .models.output_heads import PolicyHead
if isinstance(self.model.output_head, PolicyHead):
# Doing the backward pass manually, since there might not be a loss
# at each step.
self.trainer_options.automatic_optimization = False
self.trainer = self.create_trainer(setting)
self.setting = setting
# setting = DomainIncrementalSetting(
# dataset="mnist", nb_tasks=5, monitor_training_performance=True
# )
# - "Medium": Class-Incremental MNIST Setting, useful for quick debugging:
# setting = ClassIncrementalSetting(
# dataset="mnist",
# nb_tasks=5,
# monitor_training_performance=True,
# known_task_boundaries_at_test_time=False,
# batch_size=32,
# num_workes=4,
# )
# - "HARD": Class-Incremental Synbols, more challenging.
# NOTE: This Setting is very similar to the one used for the SL track of the
# competition.
setting = ClassIncrementalSetting(
dataset="synbols",
nb_tasks=12,
known_task_boundaries_at_test_time=False,
monitor_training_performance=True,
batch_size=32,
num_workers=4,
)
# Run the experiment:
results = setting.apply(method,
config=Config(debug=True, data_dir="./data"))
print(results.summary())
# from sequoia.settings.sl.class_incremental.domain_incremental import DomainIncrementalSLSetting
# setting = DomainIncrementalSLSetting(
# dataset="mnist", nb_tasks=5, monitor_training_performance=True
# )
# - "Medium": Class-Incremental MNIST Setting, useful for quick debugging:
# setting = ClassIncrementalSetting(
# dataset="mnist",
# nb_tasks=5,
# monitor_training_performance=True,
# known_task_boundaries_at_test_time=False,
# batch_size=32,
# num_workes=4,
# )
# - "HARD": Class-Incremental Synbols, more challenging.
# NOTE: This Setting is very similar to the one used for the SL track of the
# competition.
setting = ClassIncrementalSetting(
dataset="synbols",
nb_tasks=12,
known_task_boundaries_at_test_time=False,
monitor_training_performance=True,
batch_size=32,
num_workers=4,
)
# Run the experiment:
results = setting.apply(method, config=Config(debug=True, data_dir="./data"))
print(results.summary())