def main_sl():
""" Applies the PnnMethod in a SL Setting. """
parser = ArgumentParser(description=__doc__,
add_dest_to_option_strings=False)
# Add arguments for the Setting
# TODO: PNN is coded for the DomainIncrementalSetting, where the action space
# is the same for each task.
# parser.add_arguments(DomainIncrementalSetting, dest="setting")
parser.add_arguments(TaskIncrementalSLSetting, dest="setting")
# TaskIncrementalSLSetting.add_argparse_args(parser, dest="setting")
Config.add_argparse_args(parser, dest="config")
# Add arguments for the Method:
PnnMethod.add_argparse_args(parser, dest="method")
args = parser.parse_args()
# setting: TaskIncrementalSLSetting = args.setting
setting: TaskIncrementalSLSetting = TaskIncrementalSLSetting.from_argparse_args(
# setting: DomainIncrementalSetting = DomainIncrementalSetting.from_argparse_args(
args,
dest="setting",
)
config: Config = Config.from_argparse_args(args, dest="config")
method: PnnMethod = PnnMethod.from_argparse_args(args, dest="method")
method.config = config
results = setting.apply(method, config=config)
print(results.summary())
return results
def apply(
self, method: Method, config: Config = None
) -> "ContinualRLSetting.Results":
"""Apply the given method on this setting to producing some results. """
# Use the supplied config, or parse one from the arguments that were
# used to create `self`.
self.config: Config
if config is not None:
self.config = config
logger.debug(f"Using Config {self.config}")
elif isinstance(getattr(method, "config", None), Config):
self.config = method.config
logger.debug(f"Using Config from the Method: {self.config}")
else:
logger.debug(f"Parsing the Config from the command-line.")
self.config = Config.from_args(self._argv, strict=False)
logger.debug(f"Resulting Config: {self.config}")
# TODO: Test to make sure that this doesn't cause any other bugs with respect to
# the display of stuff:
# Call this method, which creates a virtual display if necessary.
self.config.get_display()
# TODO: Should we really overwrite the method's 'config' attribute here?
if not getattr(method, "config", None):
method.config = self.config
# TODO: Remove `Setting.configure(method)` entirely, from everywhere,
# and use the `prepare_data` or `setup` methods instead (since these
# `configure` methods aren't using the `method` anyway.)
method.configure(setting=self)
# BUG This won't work if the task schedule uses callables as the values (as
# they aren't json-serializable.)
if self._new_random_task_on_reset:
logger.info(
f"Train tasks: "
+ json.dumps(list(self.train_task_schedule.values()), indent="\t")
)
else:
logger.info(
f"Train task schedule:"
+ json.dumps(self.train_task_schedule, indent="\t")
)
if self.config.debug:
logger.debug(
f"Test task schedule:"
+ json.dumps(self.test_task_schedule, indent="\t")
)
# Run the Training loop (which is defined in IncrementalSetting).
results = self.main_loop(method)
logger.info("Results summary:")
logger.info(results.to_log_dict())
logger.info(results.summary())
method.receive_results(self, results=results)
return results
def main_rl():
""" Applies the PnnMethod in a RL Setting. """
parser = ArgumentParser(description=__doc__,
add_dest_to_option_strings=False)
Config.add_argparse_args(parser, dest="config")
PnnMethod.add_argparse_args(parser, dest="method")
# Haven't tested with observe_state_directly=False
# it run but I don't know if it converge
setting = TaskIncrementalRLSetting(
dataset="cartpole",
observe_state_directly=True,
nb_tasks=2,
train_task_schedule={
0: {
"gravity": 10,
"length": 0.3
},
1000: {
"gravity": 10,
"length": 0.5
},
},
)
args = parser.parse_args()
config: Config = Config.from_argparse_args(args, dest="config")
method: PnnMethod = PnnMethod.from_argparse_args(args, dest="method")
method.config = config
# 2. Creating the Method
# method = ImproveMethod()
# 3. Applying the method to the setting:
results = setting.apply(method, config=config)
print(results.summary())
print(f"objective: {results.objective}")
return results
def test_last_batch_baseline_model():
""" BUG: Baseline method is doing something weird at the last batch, and I dont know quite why.
"""
n_samples = 110
batch_size = 20
# Note: the y's here are different.
dataset = TensorDataset(
torch.arange(n_samples).reshape([n_samples, 1, 1, 1]) *
torch.ones([n_samples, 3, 32, 32]),
torch.zeros(n_samples, dtype=int),
)
pretend_to_be_active = False
env = PassiveEnvironment(
dataset,
batch_size=batch_size,
n_classes=n_samples,
pretend_to_be_active=pretend_to_be_active,
)
env = TypedObjectsWrapper(env,
observations_type=Observations,
actions_type=Actions,
rewards_type=Rewards)
env = MeasureSLPerformanceWrapper(env, first_epoch_only=True)
setting = ClassIncrementalSetting()
setting.train_env = env
model = BaselineModel(setting=setting,
hparams=BaselineModel.HParams(),
config=Config(debug=True))
for i, (obs, rew) in enumerate(env):
# assert rew is None
# if i != 5:
# assert obs.batch_size == 20, i
# else:
# assert obs.batch_size == 10, i
# actions = Actions(y_pred=torch.arange(i * 20 , (i+1) * 20)[:obs.batch_size])
# rewards = env.send(actions)
# assert (rewards.y == torch.arange(i * 20 , (i+1) * 20)[:obs.batch_size]).all()
obs = dataclasses.replace(obs,
task_labels=torch.ones([obs.x.shape[0]],
device=obs.x.device))
assert rew is None
stuff = model.training_step((obs, rew), batch_idx=i)
print(stuff)
perf = env.get_average_online_performance()
assert perf.n_samples == 110
def demo_simple():
""" Simple demo: Creating and applying a Method onto a Setting. """
from sequoia.settings import DomainIncrementalSetting
## 1. Creating the setting:
setting = DomainIncrementalSetting(dataset="fashionmnist", batch_size=32)
## 2. Creating the Method
method = DemoMethod()
# (Optional): You can also create a Config, which holds other fields like
# `log_dir`, `debug`, `device`, etc. which aren't specific to either the
# Setting or the Method.
config = Config(debug=True, render=False)
## 3. Applying the method to the setting: (optionally passing a Config to
# use for that run)
results = setting.apply(method, config=config)
print(results.summary())
print(f"objective: {results.objective}")
def baseline_demo_simple():
config = Config()
method = BaselineMethod(config=config, max_epochs=1)
## Create *any* Setting from the tree, for example:
## Supervised Learning Setting:
# setting = TaskIncrementalSLSetting(
# dataset="cifar10",
# nb_tasks=2,
# )
# Reinforcement Learning Setting:
setting = TaskIncrementalRLSetting(
dataset="cartpole",
max_steps=4000,
nb_tasks=2,
)
results = setting.apply(method, config=config)
print(results.summary())
return results
def prepare_data(self, *args, **kwargs) -> None:
# We don't really download anything atm.
if self.config is None:
self.config = Config()
super().prepare_data(*args, **kwargs)
y_pred = action_space.sample()
return self.target_setting.Actions(y_pred)
if __name__ == "__main__":
from sequoia.common import Config
from sequoia.settings import ClassIncrementalSetting
# Create the Method:
# - Manually:
method = DummyMethod()
# NOTE: This Setting is very similar to the one used for the SL track of the
# competition.
from sequoia.client import SettingProxy
setting = SettingProxy(ClassIncrementalSetting, "sl_track")
# setting = SettingProxy(ClassIncrementalSetting,
# dataset="synbols",
# nb_tasks=12,
# known_task_boundaries_at_test_time=False,
# monitor_training_performance=True,
# batch_size=32,
# num_workers=4,
# )
# NOTE: can also use pass a `Config` object to `setting.apply`. This object has some
# configuration options like device, data_dir, etc.
results = setting.apply(method, config=Config(data_dir="data"))
print(results.summary())
def __init__(
self,
hparams: BaselineModel.HParams = None,
config: Config = None,
trainer_options: TrainerConfig = None,
**kwargs,
):
""" Creates a new BaselineMethod, using the provided configuration options.
Parameters
----------
hparams : BaselineModel.HParams, optional
Hyper-parameters of the BaselineModel used by this Method. Defaults to None.
config : Config, optional
Configuration dataclass with options like log_dir, device, etc. Defaults to
None.
trainer_options : TrainerConfig, optional
Dataclass which holds all the options for creating the `pl.Trainer` which
will be used for training. Defaults to None.
**kwargs :
If any of the above arguments are left as `None`, then they will be created
using any appropriate value from `kwargs`, if present.
## Examples:
```
method = BaselineMethod(hparams=BaselineModel.HParams(learning_rate=0.01))
method = BaselineMethod(learning_rate=0.01) # Same as above
method = BaselineMethod(config=Config(debug=True))
method = BaselineMethod(debug=True) # Same as above
method = BaselineMethod(hparams=BaselineModel.HParams(learning_rate=0.01),
config=Config(debug=True))
method = BaselineMethod(learning_rate=0.01, debug=True) # Same as above
```
"""
# TODO: When creating a Method from a script, like `BaselineMethod()`,
# should we expect the hparams to be passed? Should we create them from
# the **kwargs? Should we parse them from the command-line?
# Option 2: Try to use the keyword arguments to create the hparams,
# config and trainer options.
if kwargs:
logger.info(
f"using keyword arguments {kwargs} to populate the corresponding "
f"values in the hparams, config and trainer_options.")
self.hparams = hparams or BaselineModel.HParams.from_dict(
kwargs, drop_extra_fields=True)
self.config = config or Config.from_dict(kwargs,
drop_extra_fields=True)
self.trainer_options = trainer_options or TrainerConfig.from_dict(
kwargs, drop_extra_fields=True)
elif self._argv:
# Since the method was parsed from the command-line, parse those as
# well from the argv that were used to create the Method.
# Option 3: Parse them from the command-line.
# assert not kwargs, "Don't pass any extra kwargs to the constructor!"
self.hparams = hparams or BaselineModel.HParams.from_args(
self._argv, strict=False)
self.config = config or Config.from_args(self._argv, strict=False)
self.trainer_options = trainer_options or TrainerConfig.from_args(
self._argv, strict=False)
else:
# Option 1: Use the default values:
self.hparams = hparams or BaselineModel.HParams()
self.config = config or Config()
self.trainer_options = trainer_options or TrainerConfig()
assert self.hparams
assert self.config
assert self.trainer_options
if self.config.debug:
# Disable wandb logging if debug is True.
self.trainer_options.no_wandb = True
# The model and Trainer objects will be created in `self.configure`.
# NOTE: This right here doesn't create the fields, it just gives some
# type information for static type checking.
self.trainer: Trainer
self.model: BaselineModel
self.additional_train_wrappers: List[Callable] = []
self.additional_valid_wrappers: List[Callable] = []
self.setting: Setting
def configure(self, setting: SettingType) -> None:
"""Configures the method for the given Setting.
Concretely, this creates the model and Trainer objects which will be
used to train and test a model for the given `setting`.
Args:
setting (SettingType): The setting the method will be evaluated on.
TODO: For the Challenge, this should be some kind of read-only proxy to the
actual Setting.
"""
# Note: this here is temporary, just tinkering with wandb atm.
method_name: str = self.get_name()
# Set the default batch size to use, depending on the kind of Setting.
if self.hparams.batch_size is None:
if isinstance(setting, ActiveSetting):
# Default batch size of 1 in RL
self.hparams.batch_size = 1
elif isinstance(setting, PassiveSetting):
self.hparams.batch_size = 32
else:
warnings.warn(
UserWarning(
f"Dont know what batch size to use by default for setting "
f"{setting}, will try 16."))
self.hparams.batch_size = 16
# Set the batch size on the setting.
setting.batch_size = self.hparams.batch_size
# TODO: Should we set the 'config' on the setting from here?
if setting.config and setting.config == self.config:
pass
elif self.config != Config():
assert (
setting.config is None or setting.config == Config()
), "method.config has been modified, and so has setting.config!"
setting.config = self.config
elif setting.config:
assert (setting.config !=
Config()), "Weird, both configs have default values.."
self.config = setting.config
setting_name: str = setting.get_name()
dataset = setting.dataset
if isinstance(setting, IncrementalSetting):
if self.hparams.multihead is None:
# Use a multi-head model by default if the task labels are
# available at both train and test time.
if setting.task_labels_at_test_time:
assert setting.task_labels_at_train_time
self.hparams.multihead = setting.task_labels_at_test_time
if isinstance(setting, ContinualRLSetting):
setting.add_done_to_observations = True
if not setting.observe_state_directly:
if self.hparams.encoder is None:
self.hparams.encoder = "simple_convnet"
# TODO: Add 'proper' transforms for cartpole, specifically?
from sequoia.common.transforms import Transforms
setting.train_transforms.append(Transforms.resize_64x64)
setting.val_transforms.append(Transforms.resize_64x64)
setting.test_transforms.append(Transforms.resize_64x64)
# Configure the baseline specifically for an RL setting.
# TODO: Select which output head to use from the command-line?
# Limit the number of epochs so we never iterate on a closed env.
# TODO: Would multiple "epochs" be possible?
if setting.max_steps is not None:
self.trainer_options.max_epochs = 1
self.trainer_options.limit_train_batches = setting.max_steps // (
setting.batch_size or 1)
self.trainer_options.limit_val_batches = min(
setting.max_steps // (setting.batch_size or 1), 1000)
# TODO: Test batch size is limited to 1 for now.
# NOTE: This isn't used, since we don't call `trainer.test()`.
self.trainer_options.limit_test_batches = setting.max_steps
self.model = self.create_model(setting)
assert self.hparams is self.model.hp
# The PolicyHead actually does its own backward pass, so we disable
# automatic optimization when using it.
from .models.output_heads import PolicyHead
if isinstance(self.model.output_head, PolicyHead):
# Doing the backward pass manually, since there might not be a loss
# at each step.
self.trainer_options.automatic_optimization = False
self.trainer = self.create_trainer(setting)
self.setting = setting
# setting = DomainIncrementalSetting(
# dataset="mnist", nb_tasks=5, monitor_training_performance=True
# )
# - "Medium": Class-Incremental MNIST Setting, useful for quick debugging:
# setting = ClassIncrementalSetting(
# dataset="mnist",
# nb_tasks=5,
# monitor_training_performance=True,
# known_task_boundaries_at_test_time=False,
# batch_size=32,
# num_workes=4,
# )
# - "HARD": Class-Incremental Synbols, more challenging.
# NOTE: This Setting is very similar to the one used for the SL track of the
# competition.
setting = ClassIncrementalSetting(
dataset="synbols",
nb_tasks=12,
known_task_boundaries_at_test_time=False,
monitor_training_performance=True,
batch_size=32,
num_workers=4,
)
# Run the experiment:
results = setting.apply(method,
config=Config(debug=True, data_dir="./data"))
print(results.summary())
# from sequoia.settings.sl.class_incremental.domain_incremental import DomainIncrementalSLSetting
# setting = DomainIncrementalSLSetting(
# dataset="mnist", nb_tasks=5, monitor_training_performance=True
# )
# - "Medium": Class-Incremental MNIST Setting, useful for quick debugging:
# setting = ClassIncrementalSetting(
# dataset="mnist",
# nb_tasks=5,
# monitor_training_performance=True,
# known_task_boundaries_at_test_time=False,
# batch_size=32,
# num_workes=4,
# )
# - "HARD": Class-Incremental Synbols, more challenging.
# NOTE: This Setting is very similar to the one used for the SL track of the
# competition.
setting = ClassIncrementalSetting(
dataset="synbols",
nb_tasks=12,
known_task_boundaries_at_test_time=False,
monitor_training_performance=True,
batch_size=32,
num_workers=4,
)
# Run the experiment:
results = setting.apply(method, config=Config(debug=True, data_dir="./data"))
print(results.summary())
def apply(self,
method: "Method",
config: Config = None) -> "SettingABC.Results":
""" Applies a Method on this experimental Setting to produce Results.
Defines the training/evaluation procedure specific to this Setting.
The training/evaluation loop can be defined however you want, as long as
it respects the following constraints:
1. This method should always return either a float or a Results object
that indicates the "performance" of this method on this setting.
2. More importantly: You **have** to make sure that you do not break
compatibility with more general methods targetting a parent setting!
It should always be the case that all methods designed for any of
this Setting's parents should also be applicable via polymorphism,
i.e., anything that is defined to work on the class `Animal` should
also work on the class `Cat`!
3. While not enforced, it is strongly encourged that you define your
training/evaluation routines at a pretty high level, so that Methods
that get applied to your Setting can make use of pytorch-lightning's
`Trainer` & `LightningDataModule` API to be neat and fast.
Parameters
----------
method : Method
A Method to apply on this Setting.
config : Optional[Config]
Optional configuration object with things like the log dir, the data
dir, cuda, wandb config, etc. When None, will be parsed from the
current command-line arguments.
Returns
-------
Results
An object that is used to measure or quantify the performance of the
Method on this experimental Setting.
"""
# For illustration purposes only:
self.config = config or Config.from_args()
method.configure(self)
# IDEA: Maybe instead of passing the train_dataloader or test_dataloader,
# objects, we could instead pass the methods of the LightningDataModule
# themselves, so we wouldn't have to configure the 'batch_size' etc
# arguments, and this way we could also directly control how many times
# the dataloader method can be called, for instance to limit the number
# of samples that a user can have access to (the number of epochs, etc).
# Or the dataloader would only allow a given number of iterations!
method.fit(
train_env=self.train_dataloader(),
valid_env=self.val_dataloader(),
)
test_metrics = []
test_environment = self.test_dataloader()
for observations in test_environment:
# Get the predictions/actions:
actions = method.get_actions(observations,
test_environment.action_space)
# Get the rewards for the given predictions.
rewards = test_environment.send(actions)
# Calculate the 'metrics' (TODO: This should be done be in the env!)
metrics = self.get_metrics(actions=actions, rewards=rewards)
test_metrics.append(metrics)
results = self.Results(test_metrics)
# TODO: allow the method to observe a 'copy' of the results, maybe?
method.receive_results(self, results)
return results