def test_publish_metric(self):
ep = EvaluationPlugin()
mval = MetricValue(self, "metric", 1.0, 0)
ep.publish_metric_value(mval)
# check key exists
assert len(ep.get_all_metrics()["metric"][1]) == 1
def test_periodic_eval(self):
model = SimpleMLP(input_size=6, hidden_size=10)
scenario = get_fast_scenario()
optimizer = SGD(model.parameters(), lr=1e-3)
criterion = CrossEntropyLoss()
curve_key = 'Top1_Acc_Stream/eval_phase/train_stream'
###################
# Case #1: No eval
###################
# we use stream acc. because it emits a single value
# for each eval loop.
acc = StreamAccuracy()
strategy = Naive(model,
optimizer,
criterion,
train_epochs=2,
eval_every=-1,
evaluator=EvaluationPlugin(acc))
strategy.train(scenario.train_stream[0])
# eval is not called in this case
assert len(strategy.evaluator.all_metrics) == 0
###################
# Case #2: Eval at the end only
###################
acc = StreamAccuracy()
strategy = Naive(model,
optimizer,
criterion,
train_epochs=2,
eval_every=0,
evaluator=EvaluationPlugin(acc))
strategy.train(scenario.train_stream[0])
# eval is called once at the end of the training loop
curve = strategy.evaluator.all_metrics[curve_key][1]
assert len(curve) == 1
###################
# Case #3: Eval after every epoch
###################
acc = StreamAccuracy()
strategy = Naive(model,
optimizer,
criterion,
train_epochs=2,
eval_every=1,
evaluator=EvaluationPlugin(acc))
strategy.train(scenario.train_stream[0])
# eval is called after every epoch + the end of the training loop
curve = strategy.evaluator.all_metrics[curve_key][1]
assert len(curve) == 3
def test_pnn(self):
# check that pnn reaches high enough accuracy.
# Ensure nothing weird is happening with the multiple heads.
set_deterministic_run(seed=0)
main_metric = StreamAccuracy()
exp_acc = ExperienceAccuracy()
evalp = EvaluationPlugin(main_metric, exp_acc, loggers=None)
model = PNN(num_layers=1, in_features=6, hidden_features_per_column=50)
optimizer = SGD(model.parameters(), lr=0.1)
strategy = PNNStrategy(
model,
optimizer,
train_mb_size=32,
device=get_device(),
eval_mb_size=512,
train_epochs=1,
evaluator=evalp,
)
benchmark = get_fast_benchmark(use_task_labels=True)
for train_batch_info in benchmark.train_stream:
strategy.train(train_batch_info)
strategy.eval(benchmark.train_stream[:])
print("TRAIN STREAM ACC: ", main_metric.result())
assert (sum(main_metric.result().values()) /
float(len(main_metric.result().keys())) > 0.5)
def main(args):
# Model getter: specify dataset and depth of the network.
model = pytorchcv_wrapper.resnet('cifar10', depth=20, pretrained=False)
# Or get a more specific model. E.g. wide resnet, with depth 40 and growth
# factor 8 for Cifar 10.
# model = pytorchcv_wrapper.get_model("wrn40_8_cifar10", pretrained=False)
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}"
if torch.cuda.is_available() and
args.cuda >= 0 else "cpu")
device = "cpu"
# --- TRANSFORMATIONS
transform = transforms.Compose([
ToTensor(),
transforms.Normalize((0.491, 0.482, 0.446), (0.247, 0.243, 0.261))
])
# --- SCENARIO CREATION
cifar_train = CIFAR10(root=expanduser("~") + "/.avalanche/data/cifar10/",
train=True, download=True, transform=transform)
cifar_test = CIFAR10(root=expanduser("~") + "/.avalanche/data/cifar10/",
train=False, download=True, transform=transform)
scenario = nc_benchmark(
cifar_train, cifar_test, 5, task_labels=False, seed=1234,
fixed_class_order=[i for i in range(10)])
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(
minibatch=True, epoch=True, experience=True, stream=True),
loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
forgetting_metrics(experience=True),
loggers=[interactive_logger])
# CREATE THE STRATEGY INSTANCE (Naive, with Replay)
cl_strategy = Naive(model, torch.optim.SGD(model.parameters(), lr=0.01),
CrossEntropyLoss(),
train_mb_size=100, train_epochs=1, eval_mb_size=100,
device=device,
plugins=[ReplayPlugin(mem_size=1000)],
evaluator=eval_plugin
)
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience ", experience.current_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def test_plugins_compatibility_checks(self):
model = SimpleMLP(input_size=6, hidden_size=10)
benchmark = get_fast_benchmark()
optimizer = SGD(model.parameters(), lr=1e-3)
criterion = CrossEntropyLoss()
evalp = EvaluationPlugin(
loss_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
loggers=[InteractiveLogger()],
strict_checks=None,
)
strategy = Naive(
model,
optimizer,
criterion,
train_epochs=2,
eval_every=-1,
evaluator=evalp,
plugins=[
EarlyStoppingPlugin(patience=10, val_stream_name="train")
],
)
strategy.train(benchmark.train_stream[0])
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}"
if torch.cuda.is_available() and
args.cuda >= 0 else "cpu")
n_batches = 5
# ---------
# --- TRANSFORMATIONS
train_transform = transforms.Compose([
RandomCrop(28, padding=4),
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
test_transform = transforms.Compose([
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
# ---------
# --- SCENARIO CREATION
mnist_train = MNIST('./data/mnist', train=True,
download=True, transform=train_transform)
mnist_test = MNIST('./data/mnist', train=False,
download=True, transform=test_transform)
scenario = nc_scenario(
mnist_train, mnist_test, n_batches, task_labels=False, seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(
minibatch=True, epoch=True, experience=True, stream=True),
loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
ExperienceForgetting(),
loggers=[interactive_logger])
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(model, torch.optim.Adam(model.parameters(), lr=0.001),
CrossEntropyLoss(),
train_mb_size=100, train_epochs=4, eval_mb_size=100, device=device,
plugins=[ReplayPlugin(mem_size=10000)],
evaluator=eval_plugin
)
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience ", experience.current_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def test_multihead_cumulative(self):
# check that multi-head reaches high enough accuracy.
# Ensure nothing weird is happening with the multiple heads.
set_deterministic_run(seed=0)
model = MHTestMLP(input_size=6, hidden_size=100)
criterion = CrossEntropyLoss()
optimizer = SGD(model.parameters(), lr=1)
main_metric = StreamAccuracy()
exp_acc = ExperienceAccuracy()
evalp = EvaluationPlugin(main_metric, exp_acc, loggers=None)
strategy = Cumulative(
model,
optimizer,
criterion,
train_mb_size=64,
device=get_device(),
eval_mb_size=512,
train_epochs=6,
evaluator=evalp,
)
benchmark = get_fast_benchmark(use_task_labels=True)
for train_batch_info in benchmark.train_stream:
strategy.train(train_batch_info)
strategy.eval(benchmark.train_stream[:])
print("TRAIN STREAM ACC: ", main_metric.result())
assert (sum(main_metric.result().values()) /
float(len(main_metric.result().keys())) > 0.7)
def test_multihead_cumulative(self):
# check that multi-head reaches high enough accuracy.
# Ensure nothing weird is happening with the multiple heads.
model = MHTestMLP(input_size=6, hidden_size=100)
criterion = CrossEntropyLoss()
optimizer = SGD(model.parameters(), lr=1)
main_metric = StreamAccuracy()
exp_acc = ExperienceAccuracy()
evalp = EvaluationPlugin(main_metric, exp_acc, loggers=None)
strategy = Cumulative(model,
optimizer,
criterion,
train_mb_size=32,
device=get_device(),
eval_mb_size=512,
train_epochs=1,
evaluator=evalp)
scenario = get_fast_scenario(use_task_labels=True)
for train_batch_info in scenario.train_stream:
strategy.train(train_batch_info)
strategy.eval(scenario.train_stream[:])
print("TRAIN STREAM ACC: ", main_metric.result())
assert main_metric.result() > 0.7
def main(args):
"""
Last Avalanche version reference performance (online):
Top1_Acc_Stream/eval_phase/test_stream = 0.9421
"""
# --- DEFAULT PARAMS ONLINE DATA INCREMENTAL LEARNING
nb_tasks = 5 # Can still design the data stream based on tasks
epochs = 1 # All data is only seen once: Online
batch_size = 10 # Only process small amount of data at a time
return_task_id = False # Data incremental (task-agnostic/task-free)
# TODO use data_incremental_generator, now experience=task
# --- CONFIG
device = torch.device(
f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0
else "cpu")
# ---------
# --- SCENARIO CREATION
scenario = SplitMNIST(nb_tasks, return_task_id=return_task_id,
fixed_class_order=[i for i in range(10)])
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=args.featsize,
hidden_size=400, hidden_layers=2)
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(experience=True, stream=True),
loss_metrics(experience=True, stream=True),
ExperienceForgetting(),
loggers=[interactive_logger])
# CoPE PLUGIN
cope = CoPEPlugin(mem_size=2000, p_size=args.featsize,
n_classes=scenario.n_classes)
# CREATE THE STRATEGY INSTANCE (NAIVE) WITH CoPE PLUGIN
cl_strategy = Naive(model, torch.optim.SGD(model.parameters(), lr=0.01),
cope.loss, # CoPE PPP-Loss
train_mb_size=batch_size, train_epochs=epochs,
eval_mb_size=100, device=device,
plugins=[cope],
evaluator=eval_plugin
)
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience ", experience.current_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}"
if torch.cuda.is_available() and
args.cuda >= 0 else "cpu")
# ---------
# --- TRANSFORMATIONS
train_transform = transforms.Compose([
RandomCrop(28, padding=4),
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
test_transform = transforms.Compose([
ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
# ---------
# --- SCENARIO CREATION
mnist_train = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=True, download=True, transform=train_transform)
mnist_test = MNIST(root=expanduser("~") + "/.avalanche/data/mnist/",
train=False, download=True, transform=test_transform)
scenario = nc_scenario(
mnist_train, mnist_test, 5, task_labels=False, seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
eval_plugin = EvaluationPlugin(
accuracy_metrics(epoch=True, experience=True, stream=True),
loss_metrics(epoch=True, experience=True, stream=True),
# save image should be False to appropriately view
# results in Interactive Logger.
# a tensor will be printed
StreamConfusionMatrix(save_image=False, normalize='all'),
loggers=InteractiveLogger()
)
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(
model, SGD(model.parameters(), lr=0.001, momentum=0.9),
CrossEntropyLoss(), train_mb_size=100, train_epochs=4, eval_mb_size=100,
device=device, evaluator=eval_plugin, plugins=[ReplayPlugin(5000)])
# TRAINING LOOP
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start of experience: ", experience.current_experience)
print("Current Classes: ", experience.classes_in_this_experience)
cl_strategy.train(experience)
print('Training completed')
print('Computing accuracy on the whole test set')
results.append(cl_strategy.eval(scenario.test_stream))
def main(args):
model = SimpleMLP(hidden_size=args.hs)
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
criterion = torch.nn.CrossEntropyLoss()
# check if selected GPU is available or use CPU
assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0."
device = torch.device(
f"cuda:{args.cuda}"
if torch.cuda.is_available() and args.cuda >= 0
else "cpu"
)
print(f"Using device: {device}")
# create Permuted MNIST scenario
scenario = PermutedMNIST(n_experiences=4)
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(
minibatch=True, epoch=True, experience=True, stream=True
),
loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
forgetting_metrics(experience=True),
loggers=[interactive_logger],
)
# create strategy
assert (
len(args.lambda_e) == 1 or len(args.lambda_e) == 5
), "Lambda_e must be a non-empty list."
lambda_e = args.lambda_e[0] if len(args.lambda_e) == 1 else args.lambda_e
strategy = LFL(
model,
optimizer,
criterion,
lambda_e=lambda_e,
train_epochs=args.epochs,
device=device,
train_mb_size=args.minibatch_size,
evaluator=eval_plugin,
)
# train on the selected scenario with the chosen strategy
print("Starting experiment...")
results = []
for train_batch_info in scenario.train_stream:
print(
"Start training on experience ", train_batch_info.current_experience
)
strategy.train(train_batch_info, num_workers=0)
print(
"End training on experience ", train_batch_info.current_experience
)
print("Computing accuracy on the test set")
results.append(strategy.eval(scenario.test_stream[:]))
def main(args):
model = SimpleMLP(hidden_size=args.hs)
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
criterion = torch.nn.CrossEntropyLoss()
# check if selected GPU is available or use CPU
assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0."
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
print(f'Using device: {device}')
# create scenario
if args.scenario == 'pmnist':
scenario = PermutedMNIST(n_experiences=args.permutations)
elif args.scenario == 'smnist':
scenario = SplitMNIST(n_experiences=5, return_task_id=False)
else:
raise ValueError("Wrong scenario name. Allowed pmnist, smnist.")
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
text_logger = TextLogger(open('log.txt', 'a'))
eval_plugin = EvaluationPlugin(accuracy_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
loss_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
ExperienceForgetting(),
loggers=[interactive_logger])
# create strategy
strategy = EWC(model,
optimizer,
criterion,
args.ewc_lambda,
args.ewc_mode,
decay_factor=args.decay_factor,
train_epochs=args.epochs,
device=device,
train_mb_size=args.minibatch_size,
evaluator=eval_plugin)
# train on the selected scenario with the chosen strategy
print('Starting experiment...')
results = []
for experience in scenario.train_stream:
print("Start training on experience ", experience.current_experience)
strategy.train(experience)
print("End training on experience", experience.current_experience)
print('Computing accuracy on the test set')
results.append(strategy.eval(scenario.test_stream[:]))
def __init__(
self,
model: Module,
optimizer,
criterion=CrossEntropyLoss(),
train_mb_size: int = 1,
train_epochs: int = 1,
eval_mb_size: int = 1,
device="cpu",
plugins=None,
evaluator=None,
eval_every=-1,
peval_mode="epoch",
):
self._criterion = criterion
self.model: Module = model
self.optimizer = optimizer
self.train_epochs: int = train_epochs
self.train_mb_size: int = train_mb_size
self.eval_mb_size: int = (
train_mb_size if eval_mb_size is None else eval_mb_size
)
self.device = device
self.plugins = [] if plugins is None else plugins
if evaluator is None:
evaluator = EvaluationPlugin()
self.plugins.append(evaluator)
self.evaluator = evaluator
""" EvaluationPlugin used for logging and metric computations. """
# Configure periodic evaluation.
assert peval_mode in {"epoch", "iteration"}
self.eval_every = eval_every
peval = PeriodicEval(eval_every, peval_mode)
self.plugins.append(peval)
self.clock = Clock()
""" Incremental counters for strategy events. """
# WARNING: Clock needs to be the last plugin, otherwise
# counters will be wrong for plugins called after it.
self.plugins.append(self.clock)
###################################################################
# State variables. These are updated during the train/eval loops. #
###################################################################
self.experience = None
self.adapted_dataset = None
self.dataloader = None
self.mbatch = None
self.mb_output = None
self.loss = None
self.is_training: bool = False
self.current_eval_stream = None
self._stop_training = False
def main(args):
model = SimpleMLP(hidden_size=args.hs)
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
criterion = torch.nn.CrossEntropyLoss()
# check if selected GPU is available or use CPU
assert args.cuda == -1 or args.cuda >= 0, "cuda must be -1 or >= 0."
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
print(f'Using device: {device}')
# create split scenario
scenario = SplitMNIST(n_experiences=5, return_task_id=False)
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(accuracy_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
loss_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
ExperienceForgetting(),
loggers=[interactive_logger])
# create strategy
assert len(args.lwf_alpha) == 1 or len(args.lwf_alpha) == 5,\
'Alpha must be a non-empty list.'
lwf_alpha = args.lwf_alpha[0] if len(
args.lwf_alpha) == 1 else args.lwf_alpha
strategy = LwF(model,
optimizer,
criterion,
alpha=lwf_alpha,
temperature=args.softmax_temperature,
train_epochs=args.epochs,
device=device,
train_mb_size=args.minibatch_size,
evaluator=eval_plugin)
# train on the selected scenario with the chosen strategy
print('Starting experiment...')
results = []
for train_batch_info in scenario.train_stream:
print("Start training on experience ",
train_batch_info.current_experience)
strategy.train(train_batch_info, num_workers=4)
print("End training on experience ",
train_batch_info.current_experience)
print('Computing accuracy on the test set')
results.append(strategy.eval(scenario.test_stream[:]))
def _test_logger(self, logp):
evalp = EvaluationPlugin(
loss_metrics(minibatch=True, epoch=True,
experience=True, stream=True),
loggers=[logp]
)
strat = Naive(self.model, self.optimizer, evaluator=evalp,
train_mb_size=32)
for e in self.benchmark.train_stream:
strat.train(e)
strat.eval(self.benchmark.train_stream)
def __init__(
self,
model: Module,
optimizer: Optimizer,
criterion=VAE_loss,
train_mb_size: int = 1,
train_epochs: int = 1,
eval_mb_size: int = None,
device=None,
plugins: Optional[List[SupervisedPlugin]] = None,
evaluator: EvaluationPlugin = EvaluationPlugin(
loggers=[InteractiveLogger()],
suppress_warnings=True,
),
eval_every=-1,
**base_kwargs
):
"""
Creates an instance of the Naive strategy.
:param model: The model.
:param optimizer: The optimizer to use.
:param criterion: The loss criterion to use.
:param train_mb_size: The train minibatch size. Defaults to 1.
:param train_epochs: The number of training epochs. Defaults to 1.
:param eval_mb_size: The eval minibatch size. Defaults to 1.
:param device: The device to use. Defaults to None (cpu).
:param plugins: Plugins to be added. Defaults to None.
:param evaluator: (optional) instance of EvaluationPlugin for logging
and metric computations.
:param eval_every: the frequency of the calls to `eval` inside the
training loop. -1 disables the evaluation. 0 means `eval` is called
only at the end of the learning experience. Values >0 mean that
`eval` is called every `eval_every` epochs and at the end of the
learning experience.
:param **base_kwargs: any additional
:class:`~avalanche.training.BaseTemplate` constructor arguments.
"""
super().__init__(
model,
optimizer,
criterion,
train_mb_size=train_mb_size,
train_epochs=train_epochs,
eval_mb_size=eval_mb_size,
device=device,
plugins=plugins,
evaluator=evaluator,
eval_every=eval_every,
**base_kwargs
)
def test_no_errors(self):
eval_plugin = EvaluationPlugin(accuracy_metrics(stream=True),
loggers=None,
benchmark=self.benchmark,
strict_checks=True)
strategy = Naive(self.model,
self.optimizer,
self.criterion,
train_epochs=2,
eval_every=0,
evaluator=eval_plugin)
for exp in self.benchmark.train_stream:
strategy.train(exp, eval_streams=[self.benchmark.test_stream])
strategy.eval(self.benchmark.test_stream)
def main(cuda: int):
# --- CONFIG
device = torch.device(
f"cuda:{cuda}" if torch.cuda.is_available() else "cpu"
)
# --- SCENARIO CREATION
scenario = SplitCIFAR10(n_experiences=2, seed=42)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes, input_size=196608 // 64)
# choose some metrics and evaluation method
eval_plugin = EvaluationPlugin(
accuracy_metrics(stream=True, experience=True),
images_samples_metrics(
on_train=True,
on_eval=True,
n_cols=10,
n_rows=10,
),
labels_repartition_metrics(
# image_creator=repartition_bar_chart_image_creator,
on_train=True,
on_eval=True,
),
loggers=[
TensorboardLogger(f"tb_data/{datetime.now()}"),
InteractiveLogger(),
],
)
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(
model,
Adam(model.parameters()),
train_mb_size=128,
train_epochs=1,
eval_mb_size=128,
device=device,
plugins=[ReplayPlugin(mem_size=1_000)],
evaluator=eval_plugin,
)
# TRAINING LOOP
for i, experience in enumerate(scenario.train_stream, 1):
cl_strategy.train(experience)
cl_strategy.eval(scenario.test_stream[:i])
def test_raise_warning(self):
eval_plugin = EvaluationPlugin(accuracy_metrics(stream=True),
loggers=None,
benchmark=self.benchmark,
strict_checks=False)
strategy = Naive(self.model,
self.optimizer,
self.criterion,
train_epochs=2,
eval_every=-1,
evaluator=eval_plugin)
for exp in self.benchmark.train_stream:
strategy.train(exp)
strategy.eval(self.benchmark.test_stream)
with self.assertWarns(UserWarning):
strategy.eval(self.benchmark.test_stream[:2])
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
# --- SCENARIO CREATION
scenario = SplitMNIST(n_experiences=10, seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
forgetting_metrics(experience=True),
loggers=[interactive_logger],
)
# CREATE THE STRATEGY INSTANCE (GenerativeReplay)
cl_strategy = GenerativeReplay(
model,
torch.optim.Adam(model.parameters(), lr=0.001),
CrossEntropyLoss(),
train_mb_size=100,
train_epochs=4,
eval_mb_size=100,
device=device,
evaluator=eval_plugin,
)
# TRAINING LOOP
print("Starting experiment...")
results = []
for experience in scenario.train_stream:
print("Start of experience ", experience.current_experience)
cl_strategy.train(experience)
print("Training completed")
print("Computing accuracy on the whole test set")
results.append(cl_strategy.eval(scenario.test_stream))
def main(args):
# Config
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
# model
model = MTSimpleMLP()
# CL Benchmark Creation
scenario = SplitMNIST(n_experiences=5, return_task_id=True)
train_stream = scenario.train_stream
test_stream = scenario.test_stream
# Prepare for training & testing
optimizer = Adam(model.parameters(), lr=0.01)
criterion = CrossEntropyLoss()
# choose some metrics and evaluation method
interactive_logger = InteractiveLogger()
eval_plugin = EvaluationPlugin(
accuracy_metrics(minibatch=False,
epoch=True,
experience=True,
stream=True),
forgetting_metrics(experience=True),
loggers=[interactive_logger],
)
# Choose a CL strategy
strategy = EWC(
model=model,
optimizer=optimizer,
criterion=criterion,
train_mb_size=128,
train_epochs=3,
eval_mb_size=128,
device=device,
evaluator=eval_plugin,
ewc_lambda=0.4,
)
# train and test loop
for train_task in train_stream:
strategy.train(train_task)
strategy.eval(test_stream)
def test_forward_callbacks(self):
# The EvaluationPlugin should forward all the callbacks to metrics,
# even those that are unused by the EvaluationPlugin itself.
class MetricMock:
def __init__(self):
self.x = 0
def before_blabla(self, strategy):
self.x += 1
met = MetricMock()
evalp = EvaluationPlugin(met)
evalp.before_blabla(None)
# it should ignore undefined callbacks
evalp.after_blabla(None)
# it should raise error for other undefined attributes
with self.assertRaises(AttributeError):
evalp.asd(None)
def run_base_online(experience, device, use_interactive_logger: bool = False):
"""
Runs OnlineNaive for one experience.
"""
# Create list of loggers to be used
loggers = []
if use_interactive_logger:
interactive_logger = InteractiveLogger()
loggers.append(interactive_logger)
# Evaluation plugin
eval_plugin = EvaluationPlugin(
accuracy_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
forgetting_metrics(experience=True),
loggers=loggers,
)
# Model
model = SimpleMLP(num_classes=10)
# Create OnlineNaive strategy
cl_strategy = OnlineNaive(
model,
torch.optim.SGD(model.parameters(), lr=0.01),
CrossEntropyLoss(),
num_passes=1,
train_mb_size=1,
device=device,
evaluator=eval_plugin,
)
start = time.time()
print("Running OnlineNaive ...")
cl_strategy.train(experience)
end = time.time()
duration = end - start
return duration
def test_pnn(self):
# check that pnn reaches high enough accuracy.
# Ensure nothing weird is happening with the multiple heads.
main_metric = StreamAccuracy()
exp_acc = ExperienceAccuracy()
evalp = EvaluationPlugin(main_metric, exp_acc, loggers=None)
strategy = PNNStrategy(1,
6,
50,
0.1,
train_mb_size=32,
device=get_device(),
eval_mb_size=512,
train_epochs=1,
evaluator=evalp)
scenario = get_fast_scenario(use_task_labels=True)
for train_batch_info in scenario.train_stream:
strategy.train(train_batch_info)
strategy.eval(scenario.train_stream[:])
print("TRAIN STREAM ACC: ", main_metric.result())
assert main_metric.result() > 0.5
def __init__(
self,
model: Module,
optimizer: Optimizer,
train_mb_size: int = 1,
train_epochs: int = 1,
eval_mb_size: Optional[int] = 1,
device="cpu",
plugins: Optional[List["SupervisedPlugin"]] = None,
evaluator: EvaluationPlugin = default_evaluator,
eval_every=-1,
peval_mode="epoch",
):
"""Init.
:param model: PyTorch model.
:param optimizer: PyTorch optimizer.
:param train_mb_size: mini-batch size for training.
:param train_epochs: number of training epochs.
:param eval_mb_size: mini-batch size for eval.
:param evaluator: (optional) instance of EvaluationPlugin for logging
and metric computations. None to remove logging.
:param eval_every: the frequency of the calls to `eval` inside the
training loop. -1 disables the evaluation. 0 means `eval` is called
only at the end of the learning experience. Values >0 mean that
`eval` is called every `eval_every` epochs and at the end of the
learning experience.
:param peval_mode: one of {'epoch', 'iteration'}. Decides whether the
periodic evaluation during training should execute every
`eval_every` epochs or iterations (Default='epoch').
"""
super().__init__(model=model, device=device, plugins=plugins)
self.optimizer: Optimizer = optimizer
""" PyTorch optimizer. """
self.train_epochs: int = train_epochs
""" Number of training epochs. """
self.train_mb_size: int = train_mb_size
""" Training mini-batch size. """
self.eval_mb_size: int = (train_mb_size
if eval_mb_size is None else eval_mb_size)
""" Eval mini-batch size. """
if evaluator is None:
evaluator = EvaluationPlugin()
self.plugins.append(evaluator)
self.evaluator = evaluator
""" EvaluationPlugin used for logging and metric computations. """
# Configure periodic evaluation.
assert peval_mode in {"epoch", "iteration"}
self.eval_every = eval_every
peval = PeriodicEval(eval_every, peval_mode)
self.plugins.append(peval)
self.clock = Clock()
""" Incremental counters for strategy events. """
# WARNING: Clock needs to be the last plugin, otherwise
# counters will be wrong for plugins called after it.
self.plugins.append(self.clock)
###################################################################
# State variables. These are updated during the train/eval loops. #
###################################################################
self.dataloader = None
""" Dataloader. """
self.mbatch = None
""" Current mini-batch. """
self.mb_output = None
""" Model's output computed on the current mini-batch. """
self.loss = None
""" Loss of the current mini-batch. """
self._stop_training = False
def __init__(self,
model: Module,
optimizer: Optimizer,
criterion,
train_mb_size: int = 1,
train_epochs: int = 1,
eval_mb_size: int = 1,
device='cpu',
plugins: Optional[Sequence['StrategyPlugin']] = None,
evaluator=default_logger):
"""
BaseStrategy is the super class of all task-based continual learning
strategies. It implements a basic training loop and callback system
that allows to execute code at each experience of the training loop.
Plugins can be used to implement callbacks to augment the training
loop with additional behavior (e.g. a memory buffer for replay).
**Scenarios**
This strategy supports several continual learning scenarios:
* class-incremental scenarios (no task labels)
* multi-task scenarios, where task labels are provided)
* multi-incremental scenarios, where the same task may be revisited
The exact scenario depends on the data stream and whether it provides
the task labels.
**Training loop**
The training loop and its callbacks are organized as follows::
train
before_training
before_training_exp
adapt_train_dataset
make_train_dataloader
before_training_epoch
before_training_iteration
before_forward
after_forward
before_backward
after_backward
after_training_iteration
before_update
after_update
after_training_epoch
after_training_exp
after_training
**Evaluation loop**
The evaluation loop and its callbacks are organized as follows::
eval
before_eval
adapt_eval_dataset
make_eval_dataloader
before_eval_exp
eval_epoch
before_eval_iteration
before_eval_forward
after_eval_forward
after_eval_iteration
after_eval_exp
after_eval
:param model: PyTorch model.
:param optimizer: PyTorch optimizer.
:param criterion: loss function.
:param train_mb_size: mini-batch size for training.
:param train_epochs: number of training epochs.
:param eval_mb_size: mini-batch size for eval.
:param device: PyTorch device where the model will be allocated.
:param plugins: (optional) list of StrategyPlugins.
:param evaluator: (optional) instance of EvaluationPlugin for logging
and metric computations. None to remove logging.
"""
self.model: Module = model
""" PyTorch model. """
self.criterion = criterion
""" Loss function. """
self.optimizer = optimizer
""" PyTorch optimizer. """
self.train_epochs: int = train_epochs
""" Number of training epochs. """
self.train_mb_size: int = train_mb_size
""" Training mini-batch size. """
self.eval_mb_size: int = train_mb_size if eval_mb_size is None \
else eval_mb_size
""" Eval mini-batch size. """
self.device = device
""" PyTorch device where the model will be allocated. """
self.plugins = [] if plugins is None else plugins
""" List of `StrategyPlugin`s. """
if evaluator is None:
evaluator = EvaluationPlugin()
self.plugins.append(evaluator)
self.evaluator = evaluator
""" EvaluationPlugin used for logging and metric computations. """
# Flow state variables
self.training_exp_counter = 0
""" Counts the number of training steps. +1 at the end of each
experience. """
self.eval_exp_id = None # eval-flow only
""" Label of the currently evaluated experience. Only at eval time. """
self.epoch: Optional[int] = None
""" Epoch counter. """
self.experience = None
""" Current experience. """
self.adapted_dataset = None
""" Data used to train. It may be modified by plugins. Plugins can
append data to it (e.g. for replay).
.. note::
This dataset may contain samples from different experiences. If you
want the original data for the current experience
use :attr:`.BaseStrategy.experience`.
"""
self.current_dataloader = None
""" Dataloader. """
self.mb_it = None
""" Iteration counter. Reset at the start of a new epoch. """
self.mb_x = None
""" Current mini-batch input. """
self.mb_y = None
""" Current mini-batch target. """
self.loss = None
""" Loss of the current mini-batch. """
self.logits = None
""" Logits computed on the current mini-batch. """
self.train_task_label: Optional[int] = None
""" Label of the current experience (train time). """
self.eval_task_label: Optional[int] = None
""" Label of the current experience (eval time). """
self.is_training: bool = False
""" True if the strategy is in training mode. """
def test_periodic_eval(self):
model = SimpleMLP(input_size=6, hidden_size=10)
model.classifier = IncrementalClassifier(model.classifier.in_features)
benchmark = get_fast_benchmark()
optimizer = SGD(model.parameters(), lr=1e-3)
criterion = CrossEntropyLoss()
curve_key = "Top1_Acc_Stream/eval_phase/train_stream/Task000"
###################
# Case #1: No eval
###################
# we use stream acc. because it emits a single value
# for each eval loop.
acc = StreamAccuracy()
strategy = Naive(
model,
optimizer,
criterion,
train_epochs=2,
eval_every=-1,
evaluator=EvaluationPlugin(acc),
)
strategy.train(benchmark.train_stream[0])
# eval is not called in this case
assert len(strategy.evaluator.get_all_metrics()) == 0
###################
# Case #2: Eval at the end only and before training
###################
acc = StreamAccuracy()
evalp = EvaluationPlugin(acc)
strategy = Naive(
model,
optimizer,
criterion,
train_epochs=2,
eval_every=0,
evaluator=evalp,
)
strategy.train(benchmark.train_stream[0])
# eval is called once at the end of the training loop
curve = strategy.evaluator.get_all_metrics()[curve_key][1]
assert len(curve) == 2
###################
# Case #3: Eval after every epoch and before training
###################
acc = StreamAccuracy()
strategy = Naive(
model,
optimizer,
criterion,
train_epochs=2,
eval_every=1,
evaluator=EvaluationPlugin(acc),
)
strategy.train(benchmark.train_stream[0])
curve = strategy.evaluator.get_all_metrics()[curve_key][1]
assert len(curve) == 3
###################
# Case #4: Eval in iteration mode
###################
acc = StreamAccuracy()
strategy = Naive(
model,
optimizer,
criterion,
train_epochs=2,
eval_every=100,
evaluator=EvaluationPlugin(acc),
peval_mode="iteration",
)
strategy.train(benchmark.train_stream[0])
curve = strategy.evaluator.get_all_metrics()[curve_key][1]
assert len(curve) == 5
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
# ---------
# --- TRANSFORMATIONS
train_transform = transforms.Compose([
RandomCrop(28, padding=4),
ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, )),
])
test_transform = transforms.Compose(
[ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))])
# ---------
# --- SCENARIO CREATION
mnist_train = MNIST(
root=expanduser("~") + "/.avalanche/data/mnist/",
train=True,
download=True,
transform=train_transform,
)
mnist_test = MNIST(
root=expanduser("~") + "/.avalanche/data/mnist/",
train=False,
download=True,
transform=test_transform,
)
scenario = nc_benchmark(mnist_train,
mnist_test,
5,
task_labels=False,
seed=1234)
# ---------
# MODEL CREATION
model = SimpleMLP(num_classes=scenario.n_classes)
interactive_logger = InteractiveLogger()
wandb_logger = WandBLogger(project_name=args.project,
run_name=args.run,
config=vars(args))
eval_plugin = EvaluationPlugin(
accuracy_metrics(
minibatch=True,
epoch=True,
epoch_running=True,
experience=True,
stream=True,
),
loss_metrics(
minibatch=True,
epoch=True,
epoch_running=True,
experience=True,
stream=True,
),
forgetting_metrics(experience=True, stream=True),
confusion_matrix_metrics(stream=True,
wandb=True,
class_names=[str(i) for i in range(10)]),
cpu_usage_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
timing_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
ram_usage_metrics(every=0.5,
minibatch=True,
epoch=True,
experience=True,
stream=True),
gpu_usage_metrics(
args.cuda,
every=0.5,
minibatch=True,
epoch=True,
experience=True,
stream=True,
),
disk_usage_metrics(minibatch=True,
epoch=True,
experience=True,
stream=True),
MAC_metrics(minibatch=True, epoch=True, experience=True),
loggers=[interactive_logger, wandb_logger],
)
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = Naive(
model,
SGD(model.parameters(), lr=0.001, momentum=0.9),
CrossEntropyLoss(),
train_mb_size=100,
train_epochs=4,
eval_mb_size=100,
device=device,
evaluator=eval_plugin,
)
# TRAINING LOOP
print("Starting experiment...")
results = []
for experience in scenario.train_stream:
print("Start of experience: ", experience.current_experience)
print("Current Classes: ", experience.classes_in_this_experience)
cl_strategy.train(experience)
print("Training completed")
print("Computing accuracy on the whole test set")
results.append(cl_strategy.eval(scenario.test_stream))
def main(args):
# Device config
device = torch.device(f"cuda:{args.cuda}"
if torch.cuda.is_available() and
args.cuda >= 0 else "cpu")
print('device ', device)
# ---------
# --- TRANSFORMATIONS
_mu = [0.485, 0.456, 0.406] # imagenet normalization
_std = [0.229, 0.224, 0.225]
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(mean=_mu,
std=_std)
])
# ---------
# --- SCENARIO CREATION
scenario = CORe50(scenario=args.scenario, train_transform=transform,
eval_transform=transform)
# ---------
eval_plugin = EvaluationPlugin(
loss_metrics(epoch=True, experience=True, stream=True),
accuracy_metrics(epoch=True, experience=True, stream=True),
forgetting_metrics(experience=True, stream=True),
loggers=[InteractiveLogger()]
)
criterion = torch.nn.CrossEntropyLoss()
model = SLDAResNetModel(device=device, arch='resnet18',
imagenet_pretrained=args.imagenet_pretrained)
# CREATE THE STRATEGY INSTANCE
cl_strategy = StreamingLDA(model, criterion,
args.feature_size, args.n_classes,
eval_mb_size=args.batch_size,
train_mb_size=args.batch_size,
train_epochs=1,
shrinkage_param=args.shrinkage,
streaming_update_sigma=args.plastic_cov,
device=device, evaluator=eval_plugin)
warnings.warn(
"The Deep SLDA example is not perfectly aligned with "
"the paper implementation since it does not use a base "
"initialization phase and instead starts streming from "
"pre-trained weights.")
# TRAINING LOOP
print('Starting experiment...')
for i, exp in enumerate(scenario.train_stream):
# fit SLDA model to batch (one sample at a time)
cl_strategy.train(exp)
# evaluate model on test data
cl_strategy.eval(scenario.test_stream)
def main(args):
# --- CONFIG
device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available()
and args.cuda >= 0 else "cpu")
# ---------
# --- TRANSFORMATIONS
train_transform = ToTensor()
test_transform = ToTensor()
# ---------
# --- SCENARIO CREATION
torch.random.manual_seed(1234)
n_exps = 100 # Keep it high to run a short exp
benchmark = split_lvis(n_experiences=n_exps,
train_transform=train_transform,
eval_transform=test_transform)
# ---------
# MODEL CREATION
# load a model pre-trained on COCO
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(
pretrained=True)
# Just tune the box predictor
for p in model.parameters():
p.requires_grad = False
# Replace the classifier with a new one, that has "num_classes" outputs
num_classes = benchmark.n_classes + 1 # N classes + background
# Get number of input features for the classifier
in_features = model.roi_heads.box_predictor.cls_score.in_features
# Replace the pre-trained head with a new one
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
model = model.to(device)
# Define the optimizer and the scheduler
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params,
lr=0.005,
momentum=0.9,
weight_decay=0.0005)
train_mb_size = 5
warmup_factor = 1.0 / 1000
warmup_iters = min(
1000,
len(benchmark.train_stream[0].dataset) // train_mb_size - 1)
lr_scheduler = torch.optim.lr_scheduler.LinearLR(
optimizer, start_factor=warmup_factor, total_iters=warmup_iters)
# CREATE THE STRATEGY INSTANCE (NAIVE)
cl_strategy = ObjectDetectionTemplate(
model=model,
optimizer=optimizer,
train_mb_size=train_mb_size,
train_epochs=1,
eval_mb_size=train_mb_size,
device=device,
plugins=[
LRSchedulerPlugin(lr_scheduler,
step_granularity='iteration',
first_exp_only=True,
first_epoch_only=True)
],
evaluator=EvaluationPlugin(timing_metrics(epoch=True),
loss_metrics(epoch_running=True),
make_lvis_metrics(),
loggers=[InteractiveLogger()]))
# TRAINING LOOP
print("Starting experiment...")
for i, experience in enumerate(benchmark.train_stream):
print("Start of experience: ", experience.current_experience)
print('Train dataset contains', len(experience.dataset), 'instances')
cl_strategy.train(experience, num_workers=8)
print("Training completed")
cl_strategy.eval(benchmark.test_stream, num_workers=8)
print('Evaluation completed')
