def test_device_of_output_head_is_correct():
""" There is a bug happening where the output head is on CPU while the rest of the
model is on GPU.
"""
setting = ClassIncrementalSetting(dataset="mnist")
method = BaselineMethod(max_epochs=1, no_wandb=True)
results = setting.apply(method)
assert 0.10 <= results.objective <= 0.30
def test_class_incremental_setting():
method = BaselineMethod(no_wandb=True, max_epochs=1)
setting = ClassIncrementalSetting()
results = setting.apply(method)
print(results.summary())
assert results.final_performance_metrics[0].n_samples == 1984
assert results.final_performance_metrics[1].n_samples == 2016
assert results.final_performance_metrics[2].n_samples == 1984
assert results.final_performance_metrics[3].n_samples == 2016
assert results.final_performance_metrics[4].n_samples == 1984
assert 0.48 <= results.final_performance_metrics[0].accuracy <= 0.55
assert 0.48 <= results.final_performance_metrics[1].accuracy <= 0.55
assert 0.60 <= results.final_performance_metrics[2].accuracy <= 0.95
assert 0.75 <= results.final_performance_metrics[3].accuracy <= 0.98
assert 0.99 <= results.final_performance_metrics[4].accuracy <= 1.00
# from sequoia.settings.sl.class_incremental.domain_incremental import DomainIncrementalSetting
# setting = DomainIncrementalSetting(
# dataset="mnist", nb_tasks=5, monitor_training_performance=True
# )
# - "Medium": Class-Incremental MNIST Setting, useful for quick debugging:
# setting = ClassIncrementalSetting(
# dataset="mnist",
# nb_tasks=5,
# monitor_training_performance=True,
# known_task_boundaries_at_test_time=False,
# batch_size=32,
# num_workers=4,
# )
# - "HARD": Class-Incremental Synbols, more challenging.
# NOTE: This Setting is very similar to the one used for the SL track of the
# competition.
setting = ClassIncrementalSetting(
dataset="synbols",
nb_tasks=12,
known_task_boundaries_at_test_time=False,
monitor_training_performance=True,
batch_size=32,
num_workers=4,
)
# NOTE: can also use pass a `Config` object to `setting.apply`. This object has some
# configuration options like device, data_dir, etc.
results = setting.apply(method, config=Config(data_dir="data"))
print(results.summary())
# setting = DomainIncrementalSetting(
# dataset="mnist", nb_tasks=5, monitor_training_performance=True
# )
# - "Medium": Class-Incremental MNIST Setting, useful for quick debugging:
# setting = ClassIncrementalSetting(
# dataset="mnist",
# nb_tasks=5,
# monitor_training_performance=True,
# known_task_boundaries_at_test_time=False,
# batch_size=32,
# num_workes=4,
# )
# - "HARD": Class-Incremental Synbols, more challenging.
# NOTE: This Setting is very similar to the one used for the SL track of the
# competition.
setting = ClassIncrementalSetting(
dataset="synbols",
nb_tasks=12,
known_task_boundaries_at_test_time=False,
monitor_training_performance=True,
batch_size=32,
num_workers=4,
)
# Run the experiment:
results = setting.apply(method,
config=Config(debug=True, data_dir="./data"))
print(results.summary())