def assert_balancing(self, policy):
benchmark = get_fast_benchmark(use_task_labels=True)
replay = ReplayPlugin(mem_size=100, storage_policy=policy)
model = SimpleMLP(num_classes=benchmark.n_classes)
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(
model,
SGD(model.parameters(), lr=0.001),
CrossEntropyLoss(),
train_mb_size=100,
train_epochs=0,
eval_mb_size=100,
plugins=[replay],
evaluator=None,
)
for exp in benchmark.train_stream:
cl_strategy.train(exp)
ext_mem = policy.buffer_groups
ext_mem_data = policy.buffer_datasets
print(list(ext_mem.keys()), [len(el) for el in ext_mem_data])
# buffer size should equal self.mem_size if data is large enough
len_tot = sum([len(el) for el in ext_mem_data])
assert len_tot == policy.max_size
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}"
if torch.cuda.is_available() and
args.cuda >= 0 else "cpu")
n_batches = 5
# ---------
# --- TRANSFORMATIONS
train_transform = transforms.Compose([
RandomCrop(28, padding=4),
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
test_transform = transforms.Compose([
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
# ---------
# --- SCENARIO CREATION
mnist_train = MNIST('./data/mnist', train=True,
download=True, transform=train_transform)
mnist_test = MNIST('./data/mnist', train=False,
download=True, transform=test_transform)
scenario = nc_scenario(
mnist_train, mnist_test, n_batches, task_labels=False, seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(
minibatch=True, epoch=True, experience=True, stream=True),
loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
ExperienceForgetting(),
loggers=[interactive_logger])
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(model, torch.optim.Adam(model.parameters(), lr=0.001),
CrossEntropyLoss(),
train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device,
plugins=[ReplayPlugin(mem_size=10000)],
evaluator=eval_plugin
)
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience ", experience.current_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def test_plugins_compatibility_checks(self):
model = SimpleMLP(input_size=6, hidden_size=10)
benchmark = get_fast_benchmark()
optimizer = SGD(model.parameters(), lr=1e-3)
criterion = CrossEntropyLoss()
evalp = EvaluationPlugin(
loss_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
loggers=[InteractiveLogger()],
strict_checks=None,
)
strategy = Naive(
model,
optimizer,
criterion,
train_epochs=2,
eval_every=-1,
evaluator=evalp,
plugins=[
EarlyStoppingPlugin(patience=10, val_stream_name="train")
],
)
strategy.train(benchmark.train_stream[0])
def test_early_stop(self):
class EarlyStopP(StrategyPlugin):
def after_training_iteration(self, strategy: 'BaseStrategy',
**kwargs):
if strategy.mb_it == 10:
strategy.stop_training()
model = SimpleMLP(input_size=6, hidden_size=100)
criterion = CrossEntropyLoss()
optimizer = SGD(model.parameters(), lr=1)
strategy = Cumulative(model,
optimizer,
criterion,
train_mb_size=1,
device=get_device(),
eval_mb_size=512,
train_epochs=1,
evaluator=None,
plugins=[EarlyStopP()])
scenario = get_fast_scenario()
for train_batch_info in scenario.train_stream:
strategy.train(train_batch_info)
assert strategy.mb_it == 11
def main(args):
# Config
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
# model
model = SimpleMLP(num_classes=10)
# CL Benchmark Creation
perm_mnist = PermutedMNIST(n_experiences=5)
train_stream = perm_mnist.train_stream
test_stream = perm_mnist.test_stream
# Prepare for training & testing
optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9)
criterion = CrossEntropyLoss()
# Joint training strategy
joint_train = JointTraining(model,
optimizer,
criterion,
train_mb_size=32,
train_epochs=1,
eval_mb_size=32,
device=device)
# train and test loop
results = []
print("Starting training.")
joint_train.train(train_stream)
results.append(joint_train.eval(test_stream))
def _test_scheduler_plugin(self, gamma, milestones, base_lr, epochs,
reset_lr, reset_scheduler, expected):
class TestPlugin(StrategyPlugin):
def __init__(self, expected_lrs):
super().__init__()
self.expected_lrs = expected_lrs
def after_training_epoch(self, strategy, **kwargs):
exp_id = strategy.training_exp_counter
expected_lr = self.expected_lrs[exp_id][strategy.epoch]
for group in strategy.optimizer.param_groups:
assert group['lr'] == expected_lr
scenario = self.create_scenario()
model = SimpleMLP(input_size=6, hidden_size=10)
optim = SGD(model.parameters(), lr=base_lr)
lrSchedulerPlugin = LRSchedulerPlugin(MultiStepLR(
optim, milestones=milestones, gamma=gamma),
reset_lr=reset_lr,
reset_scheduler=reset_scheduler)
cl_strategy = Naive(model,
optim,
CrossEntropyLoss(),
train_mb_size=32,
train_epochs=epochs,
eval_mb_size=100,
plugins=[lrSchedulerPlugin,
TestPlugin(expected)])
cl_strategy.train(scenario.train_stream[0])
cl_strategy.train(scenario.train_stream[1])
def test_early_stop(self):
class EarlyStopP(SupervisedPlugin):
def after_training_iteration(
self, strategy: "SupervisedTemplate", **kwargs
):
if strategy.clock.train_epoch_iterations == 10:
strategy.stop_training()
model = SimpleMLP(input_size=6, hidden_size=100)
criterion = CrossEntropyLoss()
optimizer = SGD(model.parameters(), lr=1)
strategy = Cumulative(
model,
optimizer,
criterion,
train_mb_size=1,
device=get_device(),
eval_mb_size=512,
train_epochs=1,
evaluator=None,
plugins=[EarlyStopP()],
)
benchmark = get_fast_benchmark()
for train_batch_info in benchmark.train_stream:
strategy.train(train_batch_info)
assert strategy.clock.train_epoch_iterations == 11
def main(args):
# Config
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
# model
model = SimpleMLP(input_size=32 * 32 * 3, num_classes=10)
# CL Benchmark Creation
scenario = SplitCIFAR10(n_experiences=5, return_task_id=True)
train_stream = scenario.train_stream
test_stream = scenario.test_stream
# Prepare for training & testing
optimizer = Adam(model.parameters(), lr=0.01)
criterion = CrossEntropyLoss()
# Choose a CL strategy
strategy = Naive(model=model,
optimizer=optimizer,
criterion=criterion,
train_mb_size=128,
train_epochs=3,
eval_mb_size=128,
device=device)
# train and test loop
for train_task in train_stream:
strategy.train(train_task, num_workers=0)
strategy.eval(test_stream)
def _test_replay_balanced_memory(self, storage_policy, mem_size):
benchmark = get_fast_benchmark(use_task_labels=True)
model = SimpleMLP(input_size=6, hidden_size=10)
replayPlugin = ReplayPlugin(
mem_size=mem_size, storage_policy=storage_policy
)
cl_strategy = Naive(
model,
SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001),
CrossEntropyLoss(),
train_mb_size=32,
train_epochs=1,
eval_mb_size=100,
plugins=[replayPlugin],
)
n_seen_data = 0
for step in benchmark.train_stream:
n_seen_data += len(step.dataset)
mem_fill = min(mem_size, n_seen_data)
cl_strategy.train(step)
lengths = []
for d in replayPlugin.storage_policy.buffer_datasets:
lengths.append(len(d))
self.assertEqual(sum(lengths), mem_fill) # Always fully filled
def test_dataload_batch_balancing(self):
scenario = get_fast_scenario()
model = SimpleMLP(input_size=6, hidden_size=10)
batch_size = 32
replayPlugin = ReplayPlugin(mem_size=20)
cl_strategy = Naive(model,
SGD(model.parameters(),
lr=0.001,
momentum=0.9,
weight_decay=0.001),
CrossEntropyLoss(),
train_mb_size=batch_size,
train_epochs=1,
eval_mb_size=100,
plugins=[replayPlugin])
for step in scenario.train_stream:
adapted_dataset = step.dataset
dataloader = MultiTaskJoinedBatchDataLoader(
adapted_dataset,
AvalancheConcatDataset(replayPlugin.ext_mem.values()),
oversample_small_tasks=True,
num_workers=0,
batch_size=batch_size,
shuffle=True)
for mini_batch in dataloader:
lengths = []
for task_id in mini_batch.keys():
lengths.append(len(mini_batch[task_id][1]))
if sum(lengths) == batch_size:
difference = max(lengths) - min(lengths)
self.assertLessEqual(difference, 1)
self.assertLessEqual(sum(lengths), batch_size)
cl_strategy.train(step)
def run_ocl_lazy_stream(experience, device):
"""
Runs simple naive strategy for one experience.
"""
model = SimpleMLP(num_classes=10).to(device)
model.train()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
criterion = torch.nn.CrossEntropyLoss()
start = time.time()
print("Running ocl_lazy_stream ...")
for exp in tqdm(fixed_size_experience_split(experience, 1)):
x, y, _ = exp.dataset[:]
x, y = x.to(device), torch.tensor([y]).to(device)
x, y = x.to(device), y.to(device)
optimizer.zero_grad()
pred = model(x)
loss = criterion(pred, y)
loss.backward()
optimizer.step()
end = time.time()
duration = end - start
return duration
def test_replay_balanced_memory(self):
scenario = self.create_scenario(task_labels=True)
mem_size = 25
model = SimpleMLP(input_size=6, hidden_size=10)
replayPlugin = ReplayPlugin(mem_size=mem_size)
cl_strategy = Naive(model,
SGD(model.parameters(),
lr=0.001,
momentum=0.9,
weight_decay=0.001),
CrossEntropyLoss(),
train_mb_size=32,
train_epochs=1,
eval_mb_size=100,
plugins=[replayPlugin])
for step in scenario.train_stream:
curr_mem_size = min(mem_size, len(step.dataset))
cl_strategy.train(step)
ext_mem = replayPlugin.ext_mem
lengths = []
for task_id in ext_mem.keys():
lengths.append(len(ext_mem[task_id]))
self.assertEqual(sum(lengths), curr_mem_size)
difference = max(lengths) - min(lengths)
self.assertLessEqual(difference, 1)
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}"
if torch.cuda.is_available() and
args.cuda >= 0 else "cpu")
# ---------
# --- TRANSFORMATIONS
train_transform = transforms.Compose([
RandomCrop(28, padding=4),
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
test_transform = transforms.Compose([
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
# ---------
# --- SCENARIO CREATION
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True, download=True, transform=train_transform)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False, download=True, transform=test_transform)
scenario = nc_scenario(
mnist_train, mnist_test, 5, task_labels=False, seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
eval_plugin = EvaluationPlugin(
accuracy_metrics(epoch=True, experience=True, stream=True),
loss_metrics(epoch=True, experience=True, stream=True),
# save image should be False to appropriately view
# results in Interactive Logger.
# a tensor will be printed
StreamConfusionMatrix(save_image=False, normalize='all'),
loggers=InteractiveLogger()
)
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(
model, SGD(model.parameters(), lr=0.001, momentum=0.9),
CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100,
device=device, evaluator=eval_plugin, plugins=[ReplayPlugin(5000)])
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience: ", experience.current_experience)
print("Current Classes: ", experience.classes_in_this_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def main(args):
"""
Last Avalanche version reference performance (online):
Top1_Acc_Stream/eval_phase/test_stream = 0.9421
"""
# --- DEFAULT PARAMS ONLINE DATA INCREMENTAL LEARNING
nb_tasks = 5 # Can still design the data stream based on tasks
epochs = 1 # All data is only seen once: Online
batch_size = 10 # Only process small amount of data at a time
return_task_id = False # Data incremental (task-agnostic/task-free)
# TODO use data_incremental_generator, now experience=task
# --- CONFIG
device = torch.device(
f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0
else "cpu")
# ---------
# --- SCENARIO CREATION
scenario = SplitMNIST(nb_tasks, return_task_id=return_task_id,
fixed_class_order=[i for i in range(10)])
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=args.featsize,
hidden_size=400, hidden_layers=2)
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(experience=True, stream=True),
loss_metrics(experience=True, stream=True),
ExperienceForgetting(),
loggers=[interactive_logger])
# CoPE PLUGIN
cope = CoPEPlugin(mem_size=2000, p_size=args.featsize,
n_classes=scenario.n_classes)
# CREATE THE STRATEGY INSTANCE (NAIVE) WITH CoPE PLUGIN
cl_strategy = Naive(model, torch.optim.SGD(model.parameters(), lr=0.01),
cope.loss, # CoPE PPP-Loss
train_mb_size=batch_size, train_epochs=epochs,
eval_mb_size=100, device=device,
plugins=[cope],
evaluator=eval_plugin
)
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience ", experience.current_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def main(args):
model = SimpleMLP(hidden_size=args.hs)
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
criterion = torch.nn.CrossEntropyLoss()
# check if selected GPU is available or use CPU
assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0."
device = torch.device(
f"cuda:{args.cuda}"
if torch.cuda.is_available() and args.cuda >= 0
else "cpu"
)
print(f"Using device: {device}")
# create Permuted MNIST scenario
scenario = PermutedMNIST(n_experiences=4)
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(
minibatch=True, epoch=True, experience=True, stream=True
),
loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
forgetting_metrics(experience=True),
loggers=[interactive_logger],
)
# create strategy
assert (
len(args.lambda_e) == 1 or len(args.lambda_e) == 5
), "Lambda_e must be a non-empty list."
lambda_e = args.lambda_e[0] if len(args.lambda_e) == 1 else args.lambda_e
strategy = LFL(
model,
optimizer,
criterion,
lambda_e=lambda_e,
train_epochs=args.epochs,
device=device,
train_mb_size=args.minibatch_size,
evaluator=eval_plugin,
)
# train on the selected scenario with the chosen strategy
print("Starting experiment...")
results = []
for train_batch_info in scenario.train_stream:
print(
"Start training on experience ", train_batch_info.current_experience
)
strategy.train(train_batch_info, num_workers=0)
print(
"End training on experience ", train_batch_info.current_experience
)
print("Computing accuracy on the test set")
results.append(strategy.eval(scenario.test_stream[:]))
def test_incremental_classifier(self):
model = SimpleMLP(input_size=6, hidden_size=10)
model.classifier = IncrementalClassifier(in_features=10)
optimizer = SGD(model.parameters(), lr=1e-3)
criterion = CrossEntropyLoss()
scenario = self.scenario
strategy = Naive(model,
optimizer,
criterion,
train_mb_size=100,
train_epochs=1,
eval_mb_size=100,
device='cpu')
strategy.evaluator.loggers = [TextLogger(sys.stdout)]
print("Current Classes: ",
scenario.train_stream[0].classes_in_this_experience)
print("Current Classes: ",
scenario.train_stream[4].classes_in_this_experience)
# train on first task
strategy.train(scenario.train_stream[0])
w_ptr = model.classifier.classifier.weight.data_ptr()
b_ptr = model.classifier.classifier.bias.data_ptr()
opt_params_ptrs = [
w.data_ptr() for group in optimizer.param_groups
for w in group['params']
]
# classifier params should be optimized
assert w_ptr in opt_params_ptrs
assert b_ptr in opt_params_ptrs
# train again on the same task.
strategy.train(scenario.train_stream[0])
# parameters should not change.
assert w_ptr == model.classifier.classifier.weight.data_ptr()
assert b_ptr == model.classifier.classifier.bias.data_ptr()
# the same classifier params should still be optimized
assert w_ptr in opt_params_ptrs
assert b_ptr in opt_params_ptrs
# update classifier with new classes.
old_w_ptr, old_b_ptr = w_ptr, b_ptr
strategy.train(scenario.train_stream[4])
opt_params_ptrs = [
w.data_ptr() for group in optimizer.param_groups
for w in group['params']
]
new_w_ptr = model.classifier.classifier.weight.data_ptr()
new_b_ptr = model.classifier.classifier.bias.data_ptr()
# weights should change.
assert old_w_ptr != new_w_ptr
assert old_b_ptr != new_b_ptr
# Old params should not be optimized. New params should be optimized.
assert old_w_ptr not in opt_params_ptrs
assert old_b_ptr not in opt_params_ptrs
assert new_w_ptr in opt_params_ptrs
assert new_b_ptr in opt_params_ptrs
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
# ---------
# --- TRANSFORMATIONS
train_transform = transforms.Compose([
RandomCrop(28, padding=4),
ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
test_transform = transforms.Compose(
[ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))])
# ---------
# --- SCENARIO CREATION
mnist_train = MNIST('./data/mnist',
train=True,
download=True,
transform=train_transform)
mnist_test = MNIST('./data/mnist',
train=False,
download=True,
transform=test_transform)
scenario = nc_scenario(mnist_train,
mnist_test,
5,
task_labels=False,
seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(model,
SGD(model.parameters(), lr=0.001, momentum=0.9),
CrossEntropyLoss(),
train_mb_size=100,
train_epochs=4,
eval_mb_size=100,
device=device)
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience: ", experience.current_experience)
print("Current Classes: ", experience.classes_in_this_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def main(args):
model = SimpleMLP(hidden_size=args.hs)
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
criterion = torch.nn.CrossEntropyLoss()
# check if selected GPU is available or use CPU
assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0."
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
print(f'Using device: {device}')
# create scenario
if args.scenario == 'pmnist':
scenario = PermutedMNIST(n_experiences=args.permutations)
elif args.scenario == 'smnist':
scenario = SplitMNIST(n_experiences=5, return_task_id=False)
else:
raise ValueError("Wrong scenario name. Allowed pmnist, smnist.")
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
text_logger = TextLogger(open('log.txt', 'a'))
eval_plugin = EvaluationPlugin(accuracy_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
loss_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
ExperienceForgetting(),
loggers=[interactive_logger])
# create strategy
strategy = EWC(model,
optimizer,
criterion,
args.ewc_lambda,
args.ewc_mode,
decay_factor=args.decay_factor,
train_epochs=args.epochs,
device=device,
train_mb_size=args.minibatch_size,
evaluator=eval_plugin)
# train on the selected scenario with the chosen strategy
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start training on experience ", experience.current_experience)
strategy.train(experience)
print("End training on experience", experience.current_experience)
print('Computing accuracy on the test set')
results.append(strategy.eval(scenario.test_stream[:]))
def main(args):
model = SimpleMLP(hidden_size=args.hs)
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
criterion = torch.nn.CrossEntropyLoss()
# check if selected GPU is available or use CPU
assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0."
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
print(f'Using device: {device}')
# create split scenario
scenario = SplitMNIST(n_experiences=5, return_task_id=False)
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(accuracy_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
loss_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
ExperienceForgetting(),
loggers=[interactive_logger])
# create strategy
assert len(args.lwf_alpha) == 1 or len(args.lwf_alpha) == 5,\
'Alpha must be a non-empty list.'
lwf_alpha = args.lwf_alpha[0] if len(
args.lwf_alpha) == 1 else args.lwf_alpha
strategy = LwF(model,
optimizer,
criterion,
alpha=lwf_alpha,
temperature=args.softmax_temperature,
train_epochs=args.epochs,
device=device,
train_mb_size=args.minibatch_size,
evaluator=eval_plugin)
# train on the selected scenario with the chosen strategy
print('Starting experiment...')
results = []
for train_batch_info in scenario.train_stream:
print("Start training on experience ",
train_batch_info.current_experience)
strategy.train(train_batch_info, num_workers=4)
print("End training on experience ",
train_batch_info.current_experience)
print('Computing accuracy on the test set')
results.append(strategy.eval(scenario.test_stream[:]))
def get_model(self, fast_test=False):
if fast_test:
model = SimpleMLP(input_size=6, hidden_size=10)
# model.classifier = IncrementalClassifier(
# model.classifier.in_features)
return model
else:
model = SimpleMLP()
# model.classifier = IncrementalClassifier(
# model.classifier.in_features)
return model
def test_periodic_eval(self):
model = SimpleMLP(input_size=6, hidden_size=10)
scenario = get_fast_scenario()
optimizer = SGD(model.parameters(), lr=1e-3)
criterion = CrossEntropyLoss()
curve_key = 'Top1_Acc_Stream/eval_phase/train_stream'
###################
# Case #1: No eval
###################
# we use stream acc. because it emits a single value
# for each eval loop.
acc = StreamAccuracy()
strategy = Naive(model,
optimizer,
criterion,
train_epochs=2,
eval_every=-1,
evaluator=EvaluationPlugin(acc))
strategy.train(scenario.train_stream[0])
# eval is not called in this case
assert len(strategy.evaluator.all_metrics) == 0
###################
# Case #2: Eval at the end only
###################
acc = StreamAccuracy()
strategy = Naive(model,
optimizer,
criterion,
train_epochs=2,
eval_every=0,
evaluator=EvaluationPlugin(acc))
strategy.train(scenario.train_stream[0])
# eval is called once at the end of the training loop
curve = strategy.evaluator.all_metrics[curve_key][1]
assert len(curve) == 1
###################
# Case #3: Eval after every epoch
###################
acc = StreamAccuracy()
strategy = Naive(model,
optimizer,
criterion,
train_epochs=2,
eval_every=1,
evaluator=EvaluationPlugin(acc))
strategy.train(scenario.train_stream[0])
# eval is called after every epoch + the end of the training loop
curve = strategy.evaluator.all_metrics[curve_key][1]
assert len(curve) == 3
def test_dataload_reinit(self):
scenario = get_fast_scenario()
model = SimpleMLP(input_size=6, hidden_size=10)
replayPlugin = ReplayPlugin(mem_size=5)
cl_strategy = Naive(
model,
SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001),
CrossEntropyLoss(), train_mb_size=16, train_epochs=1,
eval_mb_size=16,
plugins=[replayPlugin]
)
for step in scenario.train_stream[:2]:
cl_strategy.train(step)
def test_initialisation(self):
module = SimpleMLP()
module = module.to(self.device)
old_classifier_weight = torch.clone(module.classifier.weight)
old_classifier_bias = torch.clone(module.classifier.bias)
module = as_multitask(module, "classifier")
module = module.to(self.device)
new_classifier_weight = torch.clone(
module.classifier.classifiers["0"].classifier.weight)
new_classifier_bias = torch.clone(
module.classifier.classifiers["0"].classifier.bias)
self.assertTrue(
torch.equal(old_classifier_weight, new_classifier_weight))
self.assertTrue(torch.equal(old_classifier_bias, new_classifier_bias))
def main(cuda: int):
# --- CONFIG
device = torch.device(
f"cuda:{cuda}" if torch.cuda.is_available() else "cpu"
)
# --- SCENARIO CREATION
scenario = SplitCIFAR10(n_experiences=2, seed=42)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes, input_size=196608 // 64)
# choose some metrics and evaluation method
eval_plugin = EvaluationPlugin(
accuracy_metrics(stream=True, experience=True),
images_samples_metrics(
on_train=True,
on_eval=True,
n_cols=10,
n_rows=10,
),
labels_repartition_metrics(
# image_creator=repartition_bar_chart_image_creator,
on_train=True,
on_eval=True,
),
loggers=[
TensorboardLogger(f"tb_data/{datetime.now()}"),
InteractiveLogger(),
],
)
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(
model,
Adam(model.parameters()),
train_mb_size=128,
train_epochs=1,
eval_mb_size=128,
device=device,
plugins=[ReplayPlugin(mem_size=1_000)],
evaluator=eval_plugin,
)
# TRAINING LOOP
for i, experience in enumerate(scenario.train_stream, 1):
cl_strategy.train(experience)
cl_strategy.eval(scenario.test_stream[:i])
def main(args):
# Device config
device = torch.device(
f"cuda:{args.cuda}"
if torch.cuda.is_available() and args.cuda >= 0
else "cpu"
)
# model
model = SimpleMLP(num_classes=10)
# Here we show all the MNIST variation we offer in the "classic" benchmarks
if args.mnist_type == "permuted":
scenario = PermutedMNIST(n_experiences=5, seed=1)
elif args.mnist_type == "rotated":
scenario = RotatedMNIST(
n_experiences=5, rotations_list=[30, 60, 90, 120, 150], seed=1
)
else:
scenario = SplitMNIST(n_experiences=5, seed=1)
# Than we can extract the parallel train and test streams
train_stream = scenario.train_stream
test_stream = scenario.test_stream
# Prepare for training & testing
optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9)
criterion = CrossEntropyLoss()
# Continual learning strategy with default logger
cl_strategy = Naive(
model,
optimizer,
criterion,
train_mb_size=32,
train_epochs=100,
eval_mb_size=32,
device=device,
eval_every=1,
plugins=[EarlyStoppingPlugin(args.patience, "test_stream")],
)
# train and test loop
results = []
for train_task, test_task in zip(train_stream, test_stream):
print("Current Classes: ", train_task.classes_in_this_experience)
cl_strategy.train(train_task, eval_streams=[test_task])
results.append(cl_strategy.eval(test_stream))
def test_optimizer_update(self):
model = SimpleMLP()
optimizer = SGD(model.parameters(), lr=1e-3)
strategy = Naive(model, optimizer, None)
# check add_param_group
p = torch.nn.Parameter(torch.zeros(10, 10))
strategy.add_new_params_to_optimizer(p)
assert self._is_param_in_optimizer(p, strategy.optimizer)
# check new_param is in optimizer
# check old_param is NOT in optimizer
p_new = torch.nn.Parameter(torch.zeros(10, 10))
strategy.update_optimizer([p], [p_new])
assert self._is_param_in_optimizer(p_new, strategy.optimizer)
assert not self._is_param_in_optimizer(p, strategy.optimizer)
def test_outputs(self):
modules = [
(SimpleMLP(input_size=32 * 32 * 3), "classifier"),
(SimpleCNN(), "classifier"),
]
for m in modules:
self._test_outputs(*m)
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
# --- SCENARIO CREATION
scenario = SplitMNIST(n_experiences=10, seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
forgetting_metrics(experience=True),
loggers=[interactive_logger],
)
# CREATE THE STRATEGY INSTANCE (GenerativeReplay)
cl_strategy = GenerativeReplay(
model,
torch.optim.Adam(model.parameters(), lr=0.001),
CrossEntropyLoss(),
train_mb_size=100,
train_epochs=4,
eval_mb_size=100,
device=device,
evaluator=eval_plugin,
)
# TRAINING LOOP
print("Starting experiment...")
results = []
for experience in scenario.train_stream:
print("Start of experience ", experience.current_experience)
cl_strategy.train(experience)
print("Training completed")
print("Computing accuracy on the whole test set")
results.append(cl_strategy.eval(scenario.test_stream))
def test_callback_reachability(self):
# Check that all the callbacks are called during
# training and test loops.
model = SimpleMLP(input_size=6, hidden_size=10)
optimizer = SGD(model.parameters(), lr=1e-3)
criterion = CrossEntropyLoss()
scenario = self.create_scenario()
plug = MockPlugin()
strategy = Naive(model, optimizer, criterion,
train_mb_size=100, train_epochs=1, eval_mb_size=100,
device='cpu', plugins=[plug]
)
strategy.evaluator.loggers = [TextLogger(sys.stdout)]
strategy.train(scenario.train_stream[0], num_workers=4)
strategy.eval([scenario.test_stream[0]], num_workers=4)
assert all(plug.activated)
def test_multihead_optimizer_update(self):
# Check if the optimizer is updated correctly
# when heads are created and updated.
model = SimpleMLP(input_size=6, hidden_size=10)
optimizer = SGD(model.parameters(), lr=1e-3)
criterion = CrossEntropyLoss()
scenario = self.create_scenario()
plug = MultiHeadPlugin(model, 'classifier')
strategy = Naive(model,
optimizer,
criterion,
train_mb_size=100,
train_epochs=1,
eval_mb_size=100,
device='cpu',
plugins=[plug])
strategy.evaluator.loggers = [TextLogger(sys.stdout)]
print("Current Classes: ",
scenario.train_stream[0].classes_in_this_experience)
print("Current Classes: ",
scenario.train_stream[4].classes_in_this_experience)
# head creation
strategy.train(scenario.train_stream[0])
w_ptr = model.classifier.weight.data_ptr()
b_ptr = model.classifier.bias.data_ptr()
opt_params_ptrs = [
w.data_ptr() for group in optimizer.param_groups
for w in group['params']
]
assert w_ptr in opt_params_ptrs
assert b_ptr in opt_params_ptrs
# head update
strategy.train(scenario.train_stream[4])
w_ptr_new = model.classifier.weight.data_ptr()
b_ptr_new = model.classifier.bias.data_ptr()
opt_params_ptrs = [
w.data_ptr() for group in optimizer.param_groups
for w in group['params']
]
assert w_ptr not in opt_params_ptrs
assert b_ptr not in opt_params_ptrs
assert w_ptr_new in opt_params_ptrs
assert b_ptr_new in opt_params_ptrs
