def baseline_demo_command_line():
parser = ArgumentParser(__doc__, add_dest_to_option_strings=False)
# Supervised Learning Setting:
parser.add_arguments(TaskIncrementalSetting, dest="setting")
# Reinforcement Learning Setting:
parser.add_arguments(TaskIncrementalRLSetting, dest="setting")
parser.add_arguments(Config, dest="config")
BaselineMethod.add_argparse_args(parser, dest="method")
args = parser.parse_args()
setting: Setting = args.setting
config: Config = args.config
method: BaselineMethod = BaselineMethod.from_argparse_args(args, dest="method")
results = setting.apply(method, config=config)
print(results.summary())
return results
def demo_manual():
""" Apply the custom method to a Setting, creating both manually in code. """
# Create any Setting from the tree:
from sequoia.settings import TaskIncrementalRLSetting, TaskIncrementalSetting
# setting = TaskIncrementalSetting(dataset="mnist", nb_tasks=5) # SL
setting = TaskIncrementalRLSetting( # RL
dataset="cartpole",
train_task_schedule={
0: {
"gravity": 10,
"length": 0.5
},
5000: {
"gravity": 10,
"length": 1.0
},
},
observe_state_directly=True, # state input, rather than pixel input.
max_steps=10_000,
)
## Create the BaselineMethod:
config = Config(debug=True)
trainer_options = TrainerConfig(max_epochs=1)
hparams = BaselineModel.HParams()
base_method = BaselineMethod(hparams=hparams,
config=config,
trainer_options=trainer_options)
## Get the results of the baseline method:
base_results = setting.apply(base_method, config=config)
## Create the CustomMethod:
config = Config(debug=True)
trainer_options = TrainerConfig(max_epochs=1)
hparams = CustomizedBaselineModel.HParams()
new_method = CustomMethod(hparams=hparams,
config=config,
trainer_options=trainer_options)
## Get the results for the 'improved' method:
new_results = setting.apply(new_method, config=config)
print(f"\n\nComparison: BaselineMethod vs CustomMethod")
print("\n BaselineMethod results: ")
print(base_results.summary())
print("\n CustomMethod results: ")
print(new_results.summary())
def test_multi_task_setting():
method = BaselineMethod(no_wandb=True, max_epochs=1)
setting = MultiTaskSLSetting(dataset="mnist")
results = setting.apply(method)
print(results.summary())
assert results.final_performance_metrics[0].n_samples == 2112
assert results.final_performance_metrics[1].n_samples == 2016
assert results.final_performance_metrics[2].n_samples == 1888
assert results.final_performance_metrics[3].n_samples == 1984
assert results.final_performance_metrics[4].n_samples == 1984
assert 0.95 <= results.final_performance_metrics[0].accuracy
assert 0.95 <= results.final_performance_metrics[1].accuracy
assert 0.95 <= results.final_performance_metrics[2].accuracy
assert 0.95 <= results.final_performance_metrics[3].accuracy
assert 0.95 <= results.final_performance_metrics[4].accuracy
def test_class_incremental_setting():
method = BaselineMethod(no_wandb=True, max_epochs=1)
setting = ClassIncrementalSetting()
results = setting.apply(method)
print(results.summary())
assert results.final_performance_metrics[0].n_samples == 1984
assert results.final_performance_metrics[1].n_samples == 2016
assert results.final_performance_metrics[2].n_samples == 1984
assert results.final_performance_metrics[3].n_samples == 2016
assert results.final_performance_metrics[4].n_samples == 1984
assert 0.48 <= results.final_performance_metrics[0].accuracy <= 0.55
assert 0.48 <= results.final_performance_metrics[1].accuracy <= 0.55
assert 0.60 <= results.final_performance_metrics[2].accuracy <= 0.95
assert 0.75 <= results.final_performance_metrics[3].accuracy <= 0.98
assert 0.99 <= results.final_performance_metrics[4].accuracy <= 1.00
def baseline_demo_simple():
config = Config()
method = BaselineMethod(config=config, max_epochs=1)
## Create *any* Setting from the tree, for example:
## Supervised Learning Setting:
# setting = TaskIncrementalSLSetting(
# dataset="cifar10",
# nb_tasks=2,
# )
# Reinforcement Learning Setting:
setting = TaskIncrementalRLSetting(
dataset="cartpole",
max_steps=4000,
nb_tasks=2,
)
results = setting.apply(method, config=config)
print(results.summary())
return results
def demo_command_line():
""" Run the same demo as above, but customizing the Setting and Method from
the command-line.
NOTE: Remember to uncomment the function call below to use this instead of
demo_simple!
"""
## Create the `Setting` and the `Config` from the command-line, like in
## the other examples.
parser = ArgumentParser(description=__doc__)
## Add command-line arguments for any Setting in the tree:
from sequoia.settings import TaskIncrementalRLSetting, TaskIncrementalSLSetting
# parser.add_arguments(TaskIncrementalSLSetting, dest="setting")
parser.add_arguments(TaskIncrementalRLSetting, dest="setting")
parser.add_arguments(Config, dest="config")
# Add the command-line arguments for our CustomMethod (including the
# arguments for our simple regularization aux task).
CustomMethod.add_argparse_args(parser, dest="method")
args = parser.parse_args()
setting: ClassIncrementalSetting = args.setting
config: Config = args.config
# Create the BaselineMethod:
base_method = BaselineMethod.from_argparse_args(args, dest="method")
# Get the results of the BaselineMethod:
base_results = setting.apply(base_method, config=config)
## Create the CustomMethod:
new_method = CustomMethod.from_argparse_args(args, dest="method")
# Get the results for the CustomMethod:
new_results = setting.apply(new_method, config=config)
print(f"\n\nComparison: BaselineMethod vs CustomMethod:")
print(base_results.summary())
print(new_results.summary())