def demo_manual():
""" Apply the custom method to a Setting, creating both manually in code. """
# Create any Setting from the tree:
from sequoia.settings import TaskIncrementalRLSetting, TaskIncrementalSetting
# setting = TaskIncrementalSetting(dataset="mnist", nb_tasks=5) # SL
setting = TaskIncrementalRLSetting( # RL
dataset="cartpole",
train_task_schedule={
0: {
"gravity": 10,
"length": 0.5
},
5000: {
"gravity": 10,
"length": 1.0
},
},
observe_state_directly=True, # state input, rather than pixel input.
max_steps=10_000,
)
## Create the BaselineMethod:
config = Config(debug=True)
trainer_options = TrainerConfig(max_epochs=1)
hparams = BaselineModel.HParams()
base_method = BaselineMethod(hparams=hparams,
config=config,
trainer_options=trainer_options)
## Get the results of the baseline method:
base_results = setting.apply(base_method, config=config)
## Create the CustomMethod:
config = Config(debug=True)
trainer_options = TrainerConfig(max_epochs=1)
hparams = CustomizedBaselineModel.HParams()
new_method = CustomMethod(hparams=hparams,
config=config,
trainer_options=trainer_options)
## Get the results for the 'improved' method:
new_results = setting.apply(new_method, config=config)
print(f"\n\nComparison: BaselineMethod vs CustomMethod")
print("\n BaselineMethod results: ")
print(base_results.summary())
print("\n CustomMethod results: ")
print(new_results.summary())
def baseline_demo_simple():
config = Config()
method = BaselineMethod(config=config, max_epochs=1)
## Create *any* Setting from the tree, for example:
## Supervised Learning Setting:
# setting = TaskIncrementalSLSetting(
# dataset="cifar10",
# nb_tasks=2,
# )
# Reinforcement Learning Setting:
setting = TaskIncrementalRLSetting(
dataset="cartpole",
max_steps=4000,
nb_tasks=2,
)
results = setting.apply(method, config=config)
print(results.summary())
return results
def test_transforms_get_propagated():
for setting in [
TaskIncrementalRLSetting(dataset="cartpole"),
SettingProxy(TaskIncrementalRLSetting, dataset="cartpole"),
]:
assert setting.observation_space.x == Image(0, 1, shape=(3, 400, 600))
setting.train_transforms.append(Transforms.resize_64x64)
# TODO: The observation space doesn't update directly in RL whenever the
# transforms are changed.
# assert setting.observation_space.x == Image(0, 1, shape=(3, 64, 64))
assert setting.train_dataloader().reset().x.shape == (3, 64, 64)
def main_rl():
""" Applies the PnnMethod in a RL Setting. """
parser = ArgumentParser(description=__doc__,
add_dest_to_option_strings=False)
Config.add_argparse_args(parser, dest="config")
PnnMethod.add_argparse_args(parser, dest="method")
# Haven't tested with observe_state_directly=False
# it run but I don't know if it converge
setting = TaskIncrementalRLSetting(
dataset="cartpole",
observe_state_directly=True,
nb_tasks=2,
train_task_schedule={
0: {
"gravity": 10,
"length": 0.3
},
1000: {
"gravity": 10,
"length": 0.5
},
},
)
args = parser.parse_args()
config: Config = Config.from_argparse_args(args, dest="config")
method: PnnMethod = PnnMethod.from_argparse_args(args, dest="method")
method.config = config
# 2. Creating the Method
# method = ImproveMethod()
# 3. Applying the method to the setting:
results = setting.apply(method, config=config)
print(results.summary())
print(f"objective: {results.objective}")
return results
def demo():
""" Runs the EwcMethod on a simple setting, just to check that it works fine.
"""
# Adding arguments for each group directly:
parser = ArgumentParser(description=__doc__)
EwcMethod.add_argparse_args(parser, dest="method")
parser.add_arguments(Config, "config")
args = parser.parse_args()
method = EwcMethod.from_argparse_args(args, dest="method")
config: Config = args.config
task_schedule = {
0: {
"gravity": 10,
"length": 0.2
},
1000: {
"gravity": 100,
"length": 1.2
},
# 2000: {"gravity": 10, "length": 0.2},
}
setting = TaskIncrementalRLSetting(
dataset="cartpole",
train_task_schedule=task_schedule,
test_task_schedule=task_schedule,
observe_state_directly=True,
# max_steps=1000,
)
# from sequoia.settings import TaskIncrementalSetting, ClassIncrementalSetting
# setting = ClassIncrementalSetting(dataset="mnist", nb_tasks=5)
# setting = TaskIncrementalSetting(dataset="mnist", nb_tasks=5)
results = setting.apply(method, config=config)
print(results.summary())
def test_action_space_always_matches_obs_batch_size_in_RL(config: Config):
""" Same test as above, but in RL. """
from sequoia.settings import TaskIncrementalRLSetting
nb_tasks = 2
batch_size = 1
setting = TaskIncrementalRLSetting(
dataset="cartpole",
observe_state_directly=True,
nb_tasks=nb_tasks,
batch_size=batch_size,
steps_per_task=100,
test_steps_per_task=100,
num_workers=4, # Intentionally wrong
# monitor_training_performance=True, # This is still a TODO in RL.
)
# 500 "examples" in the test dataloader, since 5 * 100 steps per task..
total_samples = len(setting.test_dataloader())
method = OtherDummyMethod()
results = setting.apply(method, config=config)
expected_encountered_batch_sizes = {batch_size or 1}
last_batch_size = total_samples % (batch_size or 1)
if last_batch_size != 0:
expected_encountered_batch_sizes.add(last_batch_size)
assert set(method.batch_sizes) == expected_encountered_batch_sizes
# NOTE: Multiply by nb_tasks because the test loop is ran after each training task.
actual_num_batches = len(method.batch_sizes)
expected_num_batches = math.ceil(total_samples /
(batch_size or 1)) * nb_tasks
# MINOR BUG: There's an extra batch for each task. Might make sense, or might not,
# not sure.
assert actual_num_batches == expected_num_batches + nb_tasks
expected_total = total_samples * nb_tasks
actual_total_obs = sum(method.batch_sizes)
assert actual_total_obs == expected_total + nb_tasks
else:
raise NotImplementedError
self.model.policy.on_task_switch(task_id,
dataloader,
method=rl_method)
if __name__ == "__main__":
setting = TaskIncrementalRLSetting(
dataset="cartpole",
observe_state_directly=True,
nb_tasks=2,
train_task_schedule={
0: {
"gravity": 10,
"length": 0.3
},
1000: {
"gravity": 10,
"length": 0.5
}, # second task is 'easier' than the first one.
},
max_steps=2000,
)
method = EWCExampleMethod(reg_coefficient=0.)
results_without_reg = setting.apply(method)
method = EWCExampleMethod(reg_coefficient=100)
results_with_reg = setting.apply(method)
print("-" * 40)
print("WITHOUT EWC ")
print(results_without_reg.summary())
print(f"With EWC (coefficient={method.reg_coefficient}):")