def _test_scheduler_plugin(self, gamma, milestones, base_lr, epochs,
reset_lr, reset_scheduler, expected):
class TestPlugin(StrategyPlugin):
def __init__(self, expected_lrs):
super().__init__()
self.expected_lrs = expected_lrs
def after_training_epoch(self, strategy, **kwargs):
exp_id = strategy.training_exp_counter
expected_lr = self.expected_lrs[exp_id][strategy.epoch]
for group in strategy.optimizer.param_groups:
assert group['lr'] == expected_lr
scenario = self.create_scenario()
model = SimpleMLP(input_size=6, hidden_size=10)
optim = SGD(model.parameters(), lr=base_lr)
lrSchedulerPlugin = LRSchedulerPlugin(MultiStepLR(
optim, milestones=milestones, gamma=gamma),
reset_lr=reset_lr,
reset_scheduler=reset_scheduler)
cl_strategy = Naive(model,
optim,
CrossEntropyLoss(),
train_mb_size=32,
train_epochs=epochs,
eval_mb_size=100,
plugins=[lrSchedulerPlugin,
TestPlugin(expected)])
cl_strategy.train(scenario.train_stream[0])
cl_strategy.train(scenario.train_stream[1])
def _test_scheduler_plugin(
benchmark,
model,
optim,
scheduler,
epochs,
reset_lr,
reset_scheduler,
expected,
criterion=None,
metric=None,
eval_on_valid_stream=False,
):
lr_scheduler_plugin = LRSchedulerPlugin(
scheduler,
reset_lr=reset_lr,
reset_scheduler=reset_scheduler,
metric=metric,
)
verifier_plugin = SchedulerPluginTestPlugin(expected)
if criterion is None:
criterion = CrossEntropyLoss()
if eval_on_valid_stream:
cl_strategy = Naive(
model,
optim,
criterion,
train_mb_size=32,
train_epochs=epochs,
eval_mb_size=100,
plugins=[lr_scheduler_plugin, verifier_plugin],
eval_every=1,
evaluator=None,
)
cl_strategy.train(
benchmark.train_stream[0],
shuffle=False,
eval_streams=[benchmark.valid_stream[0]],
)
cl_strategy.train(
benchmark.train_stream[1],
shuffle=False,
eval_streams=[benchmark.valid_stream[1]],
)
else:
cl_strategy = Naive(
model,
optim,
criterion,
train_mb_size=32,
train_epochs=epochs,
eval_mb_size=100,
plugins=[lr_scheduler_plugin, verifier_plugin],
evaluator=None,
)
cl_strategy.train(benchmark.train_stream[0], shuffle=False)
cl_strategy.train(benchmark.train_stream[1], shuffle=False)
def test_scheduler_reduce_on_plateau_plugin(self):
# Regression test for issue #858
n_epochs = 20
criterion = CrossEntropyLoss()
def _prepare_rng_critical_parts(seed=1234):
torch.random.manual_seed(seed)
return (
PluginTests.create_benchmark(seed=seed),
_PlainMLP(input_size=6, hidden_size=10),
)
self._verify_rop_tests_reproducibility(
_prepare_rng_critical_parts, n_epochs, criterion
)
# Everything is in order, now we can test the plugin support for the
# ReduceLROnPlateau scheduler!
for reset_lr, reset_scheduler in itertools.product(
(True, False), (True, False)
):
with self.subTest(
reset_lr=reset_lr, reset_scheduler=reset_scheduler
):
# First, obtain the reference (expected) lr timeline by running
# a plain PyTorch training loop with ReduceLROnPlateau.
benchmark, model = _prepare_rng_critical_parts()
model.train()
expected_lrs = []
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
for exp in benchmark.train_stream:
if reset_lr:
for group in optimizer.param_groups:
group["lr"] = 0.001
if reset_scheduler:
scheduler = ReduceLROnPlateau(optimizer)
expected_lrs.append([])
train_loss = Mean()
for epoch in range(n_epochs):
train_loss.reset()
for x, y, t in TaskBalancedDataLoader(
exp.dataset,
oversample_small_groups=True,
num_workers=0,
batch_size=32,
shuffle=False,
pin_memory=False,
):
optimizer.zero_grad()
outputs = model(x)
loss = criterion(outputs, y)
train_loss.update(loss, weight=len(x))
loss.backward()
optimizer.step()
scheduler.step(train_loss.result())
for group in optimizer.param_groups:
expected_lrs[-1].append(group["lr"])
break
# Now we have the correct timeline stored in expected_lrs.
# Let's test the plugin!
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
PluginTests._test_scheduler_plugin(
benchmark,
model,
optimizer,
scheduler,
n_epochs,
reset_lr,
reset_scheduler,
expected_lrs,
criterion=criterion,
metric="train_loss",
)
# Other tests
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
scheduler2 = MultiStepLR(optimizer, [1, 2, 3])
# The metric must be set
with self.assertRaises(Exception):
LRSchedulerPlugin(scheduler, metric=None)
# Doesn't make sense to set the metric when using a non-metric
# based scheduler (should warn)
with self.assertWarns(Warning):
LRSchedulerPlugin(scheduler2, metric="train_loss")
# Must raise an error on unsupported metric
with self.assertRaises(Exception):
LRSchedulerPlugin(scheduler, metric="cuteness")
def run_experiment(config):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
torch.manual_seed(config.seed)
torch.cuda.manual_seed(config.seed)
np.random.seed(config.seed)
random.seed(config.seed)
torch.backends.cudnn.enabled = False
torch.backends.cudnn.deterministic = True
per_pixel_mean = get_dataset_per_pixel_mean(
CIFAR100(
expanduser("~") + "/.avalanche/data/cifar100/",
train=True,
download=True,
transform=transforms.Compose([transforms.ToTensor()]),
)
)
transforms_group = dict(
eval=(
transforms.Compose(
[
transforms.ToTensor(),
lambda img_pattern: img_pattern - per_pixel_mean,
]
),
None,
),
train=(
transforms.Compose(
[
transforms.ToTensor(),
lambda img_pattern: img_pattern - per_pixel_mean,
icarl_cifar100_augment_data,
]
),
None,
),
)
train_set = CIFAR100(
expanduser("~") + "/.avalanche/data/cifar100/",
train=True,
download=True,
)
test_set = CIFAR100(
expanduser("~") + "/.avalanche/data/cifar100/",
train=False,
download=True,
)
train_set = AvalancheDataset(
train_set,
transform_groups=transforms_group,
initial_transform_group="train",
)
test_set = AvalancheDataset(
test_set,
transform_groups=transforms_group,
initial_transform_group="eval",
)
scenario = nc_benchmark(
train_dataset=train_set,
test_dataset=test_set,
n_experiences=config.nb_exp,
task_labels=False,
seed=config.seed,
shuffle=False,
fixed_class_order=config.fixed_class_order,
)
evaluator = EvaluationPlugin(
EpochAccuracy(),
ExperienceAccuracy(),
StreamAccuracy(),
loggers=[InteractiveLogger()],
)
model: IcarlNet = make_icarl_net(num_classes=100)
model.apply(initialize_icarl_net)
optim = SGD(
model.parameters(),
lr=config.lr_base,
weight_decay=config.wght_decay,
momentum=0.9,
)
sched = LRSchedulerPlugin(
MultiStepLR(optim, config.lr_milestones, gamma=1.0 / config.lr_factor)
)
strategy = ICaRL(
model.feature_extractor,
model.classifier,
optim,
config.memory_size,
buffer_transform=transforms.Compose([icarl_cifar100_augment_data]),
fixed_memory=True,
train_mb_size=config.batch_size,
train_epochs=config.epochs,
eval_mb_size=config.batch_size,
plugins=[sched],
device=device,
evaluator=evaluator,
)
for i, exp in enumerate(scenario.train_stream):
eval_exps = [e for e in scenario.test_stream][: i + 1]
strategy.train(exp, num_workers=4)
strategy.eval(eval_exps, num_workers=4)
model = model.cuda()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
optimizer = SGD(
model.parameters(),
lr=HYPER_PARAMETER[mode]["start_lr"],
weight_decay=HYPER_PARAMETER[mode]["weight_decay"],
momentum=HYPER_PARAMETER[mode]["momentum"],
)
scheduler = make_scheduler(
optimizer,
HYPER_PARAMETER[mode]["step_schedular_decay"],
HYPER_PARAMETER[mode]["schedular_step"],
)
plugin_list = [LRSchedulerPlugin(scheduler)]
cl_strategy = Naive(
model,
optimizer,
CrossEntropyLoss(),
train_mb_size=HYPER_PARAMETER[mode]["batch_size"],
train_epochs=num_epoch,
eval_mb_size=HYPER_PARAMETER[mode]["batch_size"],
evaluator=eval_plugin,
device=device,
plugins=plugin_list,
)
# TRAINING LOOP
print("Starting experiment...")
results = []
def _test_scheduler_plugin(benchmark,
model,
optim,
scheduler,
epochs,
reset_lr,
reset_scheduler,
expected,
criterion=None,
metric=None,
eval_on_valid_stream=False,
granularity='epoch',
expected_granularity='iteration',
peval_mode='epoch',
first_epoch_only=False,
first_exp_only=False,
max_exps=None):
lr_scheduler_plugin = LRSchedulerPlugin(
scheduler,
reset_lr=reset_lr,
reset_scheduler=reset_scheduler,
metric=metric,
step_granularity=granularity,
first_epoch_only=first_epoch_only,
first_exp_only=first_exp_only)
verifier_plugin = SchedulerPluginTestPlugin(
expected, expected_granularity=expected_granularity)
if criterion is None:
criterion = CrossEntropyLoss()
if eval_on_valid_stream:
cl_strategy = Naive(
model,
optim,
criterion,
train_mb_size=32,
train_epochs=epochs,
eval_mb_size=100,
plugins=[lr_scheduler_plugin, verifier_plugin],
eval_every=1,
peval_mode=peval_mode,
evaluator=None,
)
for exp_id in range(len(benchmark.train_stream)):
if max_exps is not None and exp_id >= max_exps:
break
cl_strategy.train(
benchmark.train_stream[exp_id],
shuffle=False,
eval_streams=[benchmark.valid_stream[exp_id]],
)
# cl_strategy.train(
# benchmark.train_stream[1],
# shuffle=False,
# eval_streams=[benchmark.valid_stream[1]],
# )
else:
cl_strategy = Naive(
model,
optim,
criterion,
train_mb_size=32,
train_epochs=epochs,
eval_mb_size=100,
plugins=[lr_scheduler_plugin, verifier_plugin],
peval_mode=peval_mode,
evaluator=None,
)
for exp_id in range(len(benchmark.train_stream)):
if max_exps is not None and exp_id >= max_exps:
break
cl_strategy.train(benchmark.train_stream[exp_id],
shuffle=False)