def __init__(self):
"""
Creates an instance of the StreamForgetting metric.
"""
super().__init__()
self.stream_forgetting = Mean()
"""
The average forgetting over all experiences
"""
self.forgetting = Forgetting()
"""
The general metric to compute forgetting
"""
self._current_accuracy = Accuracy()
"""
The average accuracy over the current evaluation experience
"""
self.eval_exp_id = None
"""
The current evaluation experience id
"""
self.train_exp_id = None
"""
def __init__(self):
"""
Creates an instance of the average epoch cpu usage metric.
"""
self._mean = Mean()
super(RunningEpochCPUUsage, self).__init__(reset_at='epoch',
emit_at='iteration',
mode='train')
def __init__(self):
"""
Creates an instance of the average epoch cpu usage metric.
"""
super().__init__()
self._cpu_mean = Mean()
self._epoch_cpu = CPUUsage()
def __init__(self):
"""
Creates an instance of the GenericStreamForgetting metric.
"""
super().__init__()
self.stream_forgetting = Mean()
"""
def __init__(self,
scheduler,
reset_scheduler=True,
reset_lr=True,
metric=None):
"""
Creates a ``LRSchedulerPlugin`` instance.
:param scheduler: a learning rate scheduler that can be updated through
a step() method and can be reset by setting last_epoch=0.
:param reset_scheduler: If True, the scheduler is reset at the end of
the experience. Defaults to True.
:param reset_lr: If True, the optimizer learning rate is reset to its
original value. Default to True.
:param metric: the metric to use. Must be set when using
metric-based scheduling (like ReduceLROnPlateau). Only "train_loss"
and "val_loss" are supported at the moment. Beware that,
when using "val_loss", the periodic evaluation flow must be enabled
in the strategy. By default, the `eval_every` parameter of the
base strategy is -1, which means that the validation set is never
evaluated. Set that value to 1 to obtain the correct results.
Also, when using `metric="val_loss"`, remember to pass a proper
validation stream to the strategy train method, otherwise the
periodic evaluation stream will use the training set to compute
the validation loss.
"""
super().__init__()
self.scheduler = scheduler
self.reset_scheduler = reset_scheduler
self.reset_lr = reset_lr
self.metric = metric
self.rolling_metric = Mean()
# Used to detect and manage the periodic eval phase
self._was_training = False
self._eval_train_epoch = 0
arg_names = inspect.getfullargspec(self.scheduler.step)[0]
needs_metrics = "metrics" in arg_names
if needs_metrics and self.metric is None:
raise ValueError(
"The step method of this scheduler requires a metric "
"(usually the loss) to be passed. Please set a proper "
"metric parameter when creating this plugin.")
elif (not needs_metrics) and self.metric is not None:
warnings.warn("You are passing a metric value but the scheduler"
"doesn't seem to support metrics...")
if self.metric not in [None, "train_loss", "val_loss"]:
raise ValueError('Only scheduling based on "train_loss" and '
"val_loss"
""
f"is supported at the moment (got {metric}.")
LRSchedulerPlugin._patch_lr_on_plateau(self.scheduler)
def __init__(self):
"""
Creates an instance of the GenericStreamForwardTransfer metric.
"""
super().__init__()
self.stream_forward_transfer = Mean()
"""
def test_mean(self):
metric = Mean()
self.assertEqual(metric.result(), 0)
metric.update(0.1, 1)
self.assertEqual(metric.result(), 0.1)
metric.reset()
self.assertEqual(metric.result(), 0)
class RunningEpochCPUUsage(PluginMetric[float]):
"""
The running epoch CPU usage metric.
This plugin metric only works at training time
After each iteration, the metric logs the average CPU usage up
to the current epoch iteration.
"""
def __init__(self):
"""
Creates an instance of the average epoch cpu usage metric.
"""
super().__init__()
self._cpu_mean = Mean()
self._epoch_cpu = CPUUsage()
def before_training_epoch(self, strategy) -> MetricResult:
self.reset()
def before_training_iteration(self, strategy: 'BaseStrategy') \
-> 'MetricResult':
self._epoch_cpu.update()
def after_training_iteration(self, strategy: 'BaseStrategy') \
-> None:
super().after_training_iteration(strategy)
self._epoch_cpu.update()
self._cpu_mean.update(self._epoch_cpu.result())
self._epoch_cpu.reset()
return self._package_result(strategy)
def reset(self) -> None:
self._epoch_cpu.reset()
self._cpu_mean.reset()
def result(self) -> float:
return self._cpu_mean.result()
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
cpu_usage = self.result()
metric_name = get_metric_name(self, strategy)
plot_x_position = self.get_global_counter()
return [MetricValue(self, metric_name, cpu_usage, plot_x_position)]
def __str__(self):
return "RunningCPUUsage_Epoch"
def result(self) -> Dict[LabelCat, float]:
# print(self.new_classes, self.label2mean)
rv = {
"new": sum(
(self.label2mean[label] for label in self.new_classes),
start=Mean(),
).result()
}
if not self.old_classes:
return rv
rv["old"] = sum(
(self.label2mean[label] for label in self.old_classes),
start=Mean(),
).result()
return rv
def __init__(self):
"""
Creates an instance of the standalone CPU usage metric.
By default this metric in its initial state will return a CPU usage
value of 0. The metric can be updated by using the `update` method
while the average CPU usage can be retrieved using the `result` method.
"""
self._mean_usage = Mean()
"""
The mean utility that will be used to store the average usage.
"""
self._process_handle: Optional[Process] = None
"""
The process handle, lazily initialized.
"""
self._first_update = True
"""
def update(self) -> None:
"""
Consolidates the values got from the GPU sensor.
This will store the average for retrieval through the `update` method.
The previously consolidated value will be discarded.
:return: None
"""
if self._p is None:
self._start_watch()
return None
mean_usage = Mean()
for _ in range(GpuUsage.MAX_BUFFER):
try:
queue_element = self._values_queue.popleft()
mean_usage.update(queue_element[0], queue_element[1])
except IndexError:
break
self._last_result = mean_usage.result()
def _verify_rop_tests_reproducibility(self, init_strategy, n_epochs,
criterion):
# This doesn't actually test the support for the specific scheduler
# (ReduceLROnPlateau), but it's only used to check if:
# - the same model+benchmark pair can be instantiated in a
# deterministic way.
# - the same results could be obtained in a standard training loop in a
# deterministic way.
models_rnd = []
benchmarks_rnd = []
for _ in range(2):
benchmark, model = init_strategy()
models_rnd.append(model)
benchmarks_rnd.append(benchmark)
self.assert_model_equals(*models_rnd)
self.assert_benchmark_equals(*benchmarks_rnd)
expected_lrs_rnd = []
for _ in range(2):
benchmark, model = init_strategy()
expected_lrs = []
model.train()
for exp in benchmark.train_stream:
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
expected_lrs.append([])
train_loss = Mean()
for epoch in range(n_epochs):
train_loss.reset()
for x, y, t in TaskBalancedDataLoader(
exp.dataset,
oversample_small_groups=True,
num_workers=0,
batch_size=32,
shuffle=False,
pin_memory=False,
):
optimizer.zero_grad()
outputs = model(x)
loss = criterion(outputs, y)
train_loss.update(loss, weight=len(x))
loss.backward()
optimizer.step()
for group in optimizer.param_groups:
expected_lrs[-1].append(group["lr"])
break
scheduler.step(train_loss.result())
expected_lrs_rnd.append(expected_lrs)
self.assertEqual(expected_lrs_rnd[0], expected_lrs_rnd[1])
class RunningEpochCPUUsage(CPUPluginMetric):
"""
The running epoch CPU usage metric.
This plugin metric only works at training time
After each iteration, the metric logs the average CPU usage up
to the current epoch iteration.
"""
def __init__(self):
"""
Creates an instance of the average epoch cpu usage metric.
"""
self._mean = Mean()
super(RunningEpochCPUUsage, self).__init__(reset_at='epoch',
emit_at='iteration',
mode='train')
def result(self) -> float:
return self._mean.result()
def before_training_epoch(self, strategy):
super().before_training_epoch(strategy)
self._mean.reset()
def before_training_iteration(self, strategy):
super().before_training_iteration(strategy)
self.update(strategy) # start monitoring thread
def after_training_iteration(self, strategy):
super().after_training_iteration(strategy)
self.update(strategy)
self._mean.update(self._cpu.result())
self._cpu.reset()
return self._package_result(strategy)
def __str__(self):
return "RunningCPUUsage_Epoch"
def test_scheduler_reduce_on_plateau_plugin_with_val_stream(self):
# Regression test for issue #858 (part 2)
n_epochs = 20
criterion = CrossEntropyLoss()
def _prepare_rng_critical_parts(seed=1234):
torch.random.manual_seed(seed)
initial_benchmark = PluginTests.create_benchmark(seed=seed)
val_benchmark = benchmark_with_validation_stream(
initial_benchmark, 0.3, shuffle=True
)
return (val_benchmark, _PlainMLP(input_size=6, hidden_size=10))
self._verify_rop_tests_reproducibility(
_prepare_rng_critical_parts, n_epochs, criterion
)
# Everything is in order, now we can test the plugin support for the
# ReduceLROnPlateau scheduler!
for reset_lr, reset_scheduler in itertools.product(
(True, False), (True, False)
):
with self.subTest(
reset_lr=reset_lr, reset_scheduler=reset_scheduler
):
# First, obtain the reference (expected) lr timeline by running
# a plain PyTorch training loop with ReduceLROnPlateau.
benchmark, model = _prepare_rng_critical_parts()
expected_lrs = []
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
for exp_idx, exp in enumerate(benchmark.train_stream):
expected_lrs.append([])
model.train()
if reset_lr:
for group in optimizer.param_groups:
group["lr"] = 0.001
if reset_scheduler:
scheduler = ReduceLROnPlateau(optimizer)
for epoch in range(n_epochs):
for x, y, t in TaskBalancedDataLoader(
exp.dataset,
oversample_small_groups=True,
num_workers=0,
batch_size=32,
shuffle=False,
pin_memory=False,
):
optimizer.zero_grad()
outputs = model(x)
loss = criterion(outputs, y)
loss.backward()
optimizer.step()
for group in optimizer.param_groups:
expected_lrs[-1].append(group["lr"])
break
val_loss = Mean()
val_exp = benchmark.valid_stream[exp_idx]
model.eval()
with torch.no_grad():
for x, y, t in DataLoader(
val_exp.dataset,
num_workers=0,
batch_size=100,
pin_memory=False,
):
outputs = model(x)
loss = criterion(outputs, y)
val_loss.update(loss, weight=len(x))
scheduler.step(val_loss.result())
# Now we have the correct timeline stored in expected_lrs
# Let's test the plugin!
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
PluginTests._test_scheduler_plugin(
benchmark,
model,
optimizer,
scheduler,
n_epochs,
reset_lr,
reset_scheduler,
expected_lrs,
criterion=criterion,
metric="val_loss",
eval_on_valid_stream=True,
)
def test_scheduler_reduce_on_plateau_plugin(self):
# Regression test for issue #858
n_epochs = 20
criterion = CrossEntropyLoss()
def _prepare_rng_critical_parts(seed=1234):
torch.random.manual_seed(seed)
return (
PluginTests.create_benchmark(seed=seed),
_PlainMLP(input_size=6, hidden_size=10),
)
self._verify_rop_tests_reproducibility(
_prepare_rng_critical_parts, n_epochs, criterion
)
# Everything is in order, now we can test the plugin support for the
# ReduceLROnPlateau scheduler!
for reset_lr, reset_scheduler in itertools.product(
(True, False), (True, False)
):
with self.subTest(
reset_lr=reset_lr, reset_scheduler=reset_scheduler
):
# First, obtain the reference (expected) lr timeline by running
# a plain PyTorch training loop with ReduceLROnPlateau.
benchmark, model = _prepare_rng_critical_parts()
model.train()
expected_lrs = []
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
for exp in benchmark.train_stream:
if reset_lr:
for group in optimizer.param_groups:
group["lr"] = 0.001
if reset_scheduler:
scheduler = ReduceLROnPlateau(optimizer)
expected_lrs.append([])
train_loss = Mean()
for epoch in range(n_epochs):
train_loss.reset()
for x, y, t in TaskBalancedDataLoader(
exp.dataset,
oversample_small_groups=True,
num_workers=0,
batch_size=32,
shuffle=False,
pin_memory=False,
):
optimizer.zero_grad()
outputs = model(x)
loss = criterion(outputs, y)
train_loss.update(loss, weight=len(x))
loss.backward()
optimizer.step()
scheduler.step(train_loss.result())
for group in optimizer.param_groups:
expected_lrs[-1].append(group["lr"])
break
# Now we have the correct timeline stored in expected_lrs.
# Let's test the plugin!
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
PluginTests._test_scheduler_plugin(
benchmark,
model,
optimizer,
scheduler,
n_epochs,
reset_lr,
reset_scheduler,
expected_lrs,
criterion=criterion,
metric="train_loss",
)
# Other tests
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
scheduler2 = MultiStepLR(optimizer, [1, 2, 3])
# The metric must be set
with self.assertRaises(Exception):
LRSchedulerPlugin(scheduler, metric=None)
# Doesn't make sense to set the metric when using a non-metric
# based scheduler (should warn)
with self.assertWarns(Warning):
LRSchedulerPlugin(scheduler2, metric="train_loss")
# Must raise an error on unsupported metric
with self.assertRaises(Exception):
LRSchedulerPlugin(scheduler, metric="cuteness")
def __init__(
self,
scheduler,
reset_scheduler=True,
reset_lr=True,
metric=None,
step_granularity: Literal["epoch", "iteration"] = "epoch",
first_epoch_only=False,
first_exp_only=False,
):
"""
Creates a ``LRSchedulerPlugin`` instance.
:param scheduler: a learning rate scheduler that can be updated through
a step() method and can be reset by setting last_epoch=0.
:param reset_scheduler: If True, the scheduler is reset at the end of
the experience. Defaults to True.
:param reset_lr: If True, the optimizer learning rate is reset to its
original value. Default to True.
:param metric: the metric to use. Must be set when using
metric-based scheduling (like ReduceLROnPlateau). Only "train_loss"
and "val_loss" are supported at the moment. Beware that,
when using "val_loss", the periodic evaluation flow must be enabled
in the strategy. By default, the `eval_every` parameter of the
base strategy is -1, which means that the validation set is never
evaluated. Set that value to 1 to obtain the correct results.
Also, when using `metric="val_loss"`, remember to pass a proper
validation stream to the strategy train method, otherwise the
periodic evaluation stream will use the training set to compute
the validation loss.
:param step_granularity: defines how often the scheduler's `step()`
method will be called. Defaults to 'epoch'. Valid values are
'epoch' and 'iteration'.
:param first_epoch_only: if True, the scheduler will only be stepped
in the first epoch of each training experience. This is not mutually
exclusive with `first_exp_only`: by setting both values to True,
the scheduler will be stepped only in the very first epoch of the
whole training stream.
:param first_exp_only: if True, the scheduler will only be considered
in the first training experience.
"""
super().__init__()
self.scheduler = scheduler
self.reset_scheduler = reset_scheduler
self.reset_lr = reset_lr
self.metric = metric
self.rolling_metric = Mean()
self.step_granularity = step_granularity
self.first_epoch_only = first_epoch_only
self.first_exp_only = first_exp_only
# Used to detect and manage the periodic eval phase
self._was_training = False
self._just_validated = False
self._executed_train_iteration = False
arg_names = inspect.getfullargspec(self.scheduler.step)[0]
needs_metrics = "metrics" in arg_names
if needs_metrics and self.metric is None:
raise ValueError(
"The step method of this scheduler requires a metric "
"(usually the loss) to be passed. Please set a proper "
"metric parameter when creating this plugin.")
elif (not needs_metrics) and self.metric is not None:
warnings.warn("You are passing a metric value but the scheduler"
"doesn't seem to support metrics...")
if self.metric not in [None, "train_loss", "val_loss"]:
raise ValueError('Only scheduling based on "train_loss" and '
"val_loss"
""
f"is supported at the moment (got {metric}.")
if self.step_granularity not in ["iteration", "epoch"]:
raise ValueError(
"Wrong value of step_granularity: valid values are "
'"iteration" and "epoch"')
LRSchedulerPlugin._patch_lr_on_plateau(self.scheduler)
class StreamForgetting(PluginMetric[Dict[int, float]]):
"""
The StreamForgetting metric, describing the average evaluation accuracy loss
detected over all experiences observed during training.
This plugin metric, computed over all observed experiences during training,
is the average over the difference between the accuracy result obtained
after first training on a experience and the accuracy result obtained
on the same experience at the end of successive experiences.
This metric is computed during the eval phase only.
"""
def __init__(self):
"""
Creates an instance of the StreamForgetting metric.
"""
super().__init__()
self.stream_forgetting = Mean()
"""
The average forgetting over all experiences
"""
self.forgetting = Forgetting()
"""
The general metric to compute forgetting
"""
self._current_accuracy = Accuracy()
"""
The average accuracy over the current evaluation experience
"""
self.eval_exp_id = None
"""
The current evaluation experience id
"""
self.train_exp_id = None
"""
The last encountered training experience id
"""
def reset(self) -> None:
"""
Resets the forgetting metrics.
Beware that this will also reset the initial accuracy of each
experience!
:return: None.
"""
self.forgetting.reset()
self.stream_forgetting.reset()
def reset_last_accuracy(self) -> None:
"""
Resets the last accuracy.
This will preserve the initial accuracy value of each experience.
To be used at the beginning of each eval experience.
:return: None.
"""
self.forgetting.reset_last()
def exp_update(self, k, v, initial=False):
"""
Update forgetting metric.
See `Forgetting` for more detailed information.
:param k: key to update
:param v: value associated to k
:param initial: update initial value. If False, update
last value.
"""
self.forgetting.update(k, v, initial=initial)
def exp_result(self, k=None) -> Union[float, None, Dict[int, float]]:
"""
Result for experience defined by a key.
See `Forgetting` documentation for more detailed information.
k: optional key from which compute forgetting.
"""
return self.forgetting.result(k=k)
def result(self, k=None) -> Union[float, None, Dict[int, float]]:
"""
The average forgetting over all experience.
k: optional key from which compute forgetting.
"""
return self.stream_forgetting.result()
def before_training_exp(self, strategy: 'BaseStrategy') -> None:
self.train_exp_id = strategy.experience.current_experience
def before_eval(self, strategy) -> None:
self.reset_current_accuracy()
self.stream_forgetting.reset()
def before_eval_exp(self, strategy: 'BaseStrategy') -> None:
self._current_accuracy.reset()
def after_eval_iteration(self, strategy: 'BaseStrategy') -> None:
self.eval_exp_id = strategy.experience.current_experience
self._current_accuracy.update(strategy.mb_y, strategy.logits)
def after_eval_exp(self, strategy: 'BaseStrategy') -> None:
# update experience on which training just ended
if self.train_exp_id == self.eval_exp_id:
self.exp_update(self.eval_exp_id,
self._current_accuracy.result(),
initial=True)
else:
# update other experiences
# if experience has not been encountered in training
# its value will not be considered in forgetting
self.exp_update(self.eval_exp_id, self._current_accuracy.result())
# this checks if the evaluation experience has been
# already encountered at training time
# before the last training.
# If not, forgetting should not be returned.
if self.exp_result(k=self.eval_exp_id) is not None:
exp_forgetting = self.exp_result(k=self.eval_exp_id)
self.stream_forgetting.update(exp_forgetting, weight=1)
def after_eval(self, strategy: 'BaseStrategy') -> \
'MetricResult':
return self._package_result(strategy)
def _package_result(self, strategy: 'BaseStrategy') -> \
MetricResult:
metric_value = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self.get_global_counter()
return [MetricValue(self, metric_name, metric_value, plot_x_position)]
def __str__(self):
return "StreamForgetting"
class CPUUsage(Metric[float]):
"""
The standalone CPU usage metric.
Instances of this metric compute the average CPU usage as a float value.
The metric starts tracking the CPU usage when the `update` method is called
for the first time. That is, the tracking does not start at the time the
constructor is invoked.
Calling the `update` method more than twice will update the metric to the
average usage between the first and the last call to `update`.
The result, obtained using the `result` method, is the usage computed
as stated above.
The reset method will bring the metric to its initial state. By default
this metric in its initial state will return an usage value of 0.
"""
def __init__(self):
"""
Creates an instance of the standalone CPU usage metric.
By default this metric in its initial state will return a CPU usage
value of 0. The metric can be updated by using the `update` method
while the average CPU usage can be retrieved using the `result` method.
"""
self._mean_usage = Mean()
"""
The mean utility that will be used to store the average usage.
"""
self._process_handle: Optional[Process] = None
"""
The process handle, lazily initialized.
"""
self._first_update = True
"""
An internal flag to keep track of the first call to the `update` method.
"""
def update(self) -> None:
"""
Update the running CPU usage.
For more info on how to set the starting moment see the class
description.
:return: None.
"""
if self._first_update:
self._process_handle = Process(os.getpid())
last_time = getattr(self._process_handle, '_last_sys_cpu_times', None)
utilization = self._process_handle.cpu_percent()
current_time = getattr(self._process_handle, '_last_sys_cpu_times',
None)
if self._first_update:
self._first_update = False
else:
if current_time is None or last_time is None:
warnings.warn('CPUUsage can\'t detect the elapsed time. It is '
'recommended to update avalanche to the latest '
'version.')
# Fallback, shouldn't happen
current_time = 1.0
last_time = 0.0
self._mean_usage.update(utilization, current_time - last_time)
def result(self) -> float:
"""
Retrieves the average CPU usage.
Calling this method will not change the internal state of the metric.
:return: The average CPU usage, as a float value.
"""
return self._mean_usage.result()
def reset(self) -> None:
"""
Resets the metric.
:return: None.
"""
self._mean_usage.reset()
self._process_handle = None
self._first_update = True
class LRSchedulerPlugin(StrategyPlugin):
"""Learning Rate Scheduler Plugin.
This plugin manages learning rate scheduling inside of a strategy using the
PyTorch scheduler passed to the constructor. The step() method of the
scheduler is called after each training epoch.
Metric-based schedulers (like ReduceLROnPlateau) are supported as well.
"""
def __init__(self,
scheduler,
reset_scheduler=True,
reset_lr=True,
metric=None):
"""
Creates a ``LRSchedulerPlugin`` instance.
:param scheduler: a learning rate scheduler that can be updated through
a step() method and can be reset by setting last_epoch=0.
:param reset_scheduler: If True, the scheduler is reset at the end of
the experience. Defaults to True.
:param reset_lr: If True, the optimizer learning rate is reset to its
original value. Default to True.
:param metric: the metric to use. Must be set when using
metric-based scheduling (like ReduceLROnPlateau). Only "train_loss"
and "val_loss" are supported at the moment. Beware that,
when using "val_loss", the periodic evaluation flow must be enabled
in the strategy. By default, the `eval_every` parameter of the
base strategy is -1, which means that the validation set is never
evaluated. Set that value to 1 to obtain the correct results.
Also, when using `metric="val_loss"`, remember to pass a proper
validation stream to the strategy train method, otherwise the
periodic evaluation stream will use the training set to compute
the validation loss.
"""
super().__init__()
self.scheduler = scheduler
self.reset_scheduler = reset_scheduler
self.reset_lr = reset_lr
self.metric = metric
self.rolling_metric = Mean()
# Used to detect and manage the periodic eval phase
self._was_training = False
self._eval_train_epoch = 0
arg_names = inspect.getfullargspec(self.scheduler.step)[0]
needs_metrics = "metrics" in arg_names
if needs_metrics and self.metric is None:
raise ValueError(
"The step method of this scheduler requires a metric "
"(usually the loss) to be passed. Please set a proper "
"metric parameter when creating this plugin.")
elif (not needs_metrics) and self.metric is not None:
warnings.warn("You are passing a metric value but the scheduler"
"doesn't seem to support metrics...")
if self.metric not in [None, "train_loss", "val_loss"]:
raise ValueError('Only scheduling based on "train_loss" and '
"val_loss"
""
f"is supported at the moment (got {metric}.")
LRSchedulerPlugin._patch_lr_on_plateau(self.scheduler)
def after_training_epoch(self, strategy: "BaseStrategy", **kwargs):
if self.metric == "train_loss":
self.scheduler.step(metrics=self.rolling_metric.result())
self.rolling_metric.reset()
elif self.metric != "val_loss":
self.scheduler.step()
self.rolling_metric.reset()
def after_training_exp(self, strategy: "BaseStrategy", **kwargs):
param_groups = strategy.optimizer.param_groups
base_lrs = self.scheduler.base_lrs
if self.reset_lr:
for group, lr in zip(param_groups, base_lrs):
group["lr"] = lr
if self.reset_scheduler:
self.scheduler.last_epoch = 0
# Manage the reset of the scheduler
# Mainly used to call _reset on ReduceLROnPlateau, but may come
# in handy for other schedulers in the future
reset_method = getattr(self.scheduler, "reset", None)
if not callable(reset_method):
reset_method = getattr(self.scheduler, "_reset", None)
if callable(reset_method):
# print('Calling reset method of scheduler')
reset_method()
# Methods used to manage ReduceLROnPlateau (keep track of the periodic eval)
def before_training(self, strategy: "BaseStrategy", **kwargs):
self._was_training = True
def after_training(self, strategy: "BaseStrategy", **kwargs):
self._was_training = False
def after_eval(self, strategy: "BaseStrategy", **kwargs):
if self.metric == "val_loss" and self._was_training:
if strategy.clock.train_exp_epochs == 0:
# The base strategy may run an evaluation pass on the
# validation set before running the training loop. In that
# case, we should just discard the result.
# print('Ignoring pre-training validation')
pass
elif self._eval_train_epoch == strategy.clock.train_exp_epochs:
# The base strategy may run an evaluation pass on the
# validation set after the training loop. In that
# case, we should discard the result only if the validation pass
# has been duplicated.
# In fact, the previous branch of the "if" could be omitted
# because this one can cover both the pre-training and
# duplicate post-training cases...
# print('Ignoring post-training duplicate validation '
# f'{self._eval_train_epoch}')
pass
else:
# print('Stepping after validation',
# self.rolling_metric.result())
self.scheduler.step(metrics=self.rolling_metric.result())
self.rolling_metric.reset()
self._eval_train_epoch = strategy.clock.train_exp_epochs
def after_training_iteration(self, strategy: "BaseStrategy", **kwargs):
if self.metric != "train_loss":
return
self.rolling_metric.update(strategy.loss, weight=len(strategy.mb_x))
def after_eval_iteration(self, strategy: "BaseStrategy", **kwargs):
if self.metric != "val_loss":
return
# Check if switched to eval mid-training
# This only happens when running periodic validation
if self._was_training:
self.rolling_metric.update(strategy.loss,
weight=len(strategy.mb_x))
@staticmethod
def _patch_lr_on_plateau(scheduler):
# All PyTorch schedulers have the base_lrs field (needed to reset the
# initial LRs before each experience) with the only exception being
# ReduceLROnPlateau. This method will add that field to
# ReduceLROnPlateau.
if hasattr(scheduler, "base_lrs"):
return
# Initialize epoch and base learning rates
for group in scheduler.optimizer.param_groups:
group.setdefault("initial_lr", group["lr"])
scheduler.base_lrs = list(
map(
lambda group_param: group_param["initial_lr"],
scheduler.optimizer.param_groups,
))
class GenericStreamForwardTransfer(GenericExperienceForwardTransfer):
"""
The GenericStreamForwardTransfer metric, describing the average evaluation
forward transfer detected over all experiences observed during training.
In particular, the user should override:
* __init__ by calling `super` and instantiating the `self.current_metric`
property as a valid avalanche metric
* `metric_update`, to update `current_metric`
* `metric_result` to get the result from `current_metric`.
* `__str__` to define the experience forgetting name.
This metric is computed during the eval phase only.
"""
def __init__(self):
"""
Creates an instance of the GenericStreamForwardTransfer metric.
"""
super().__init__()
self.stream_forward_transfer = Mean()
"""
The average forward transfer over all experiences
"""
def reset(self) -> None:
"""
Resets the forward transfer metrics.
Note that this will reset the previous and initial accuracy of each
experience.
:return: None.
"""
super().reset()
self.stream_forward_transfer.reset()
def exp_update(self, k, v, initial=False):
"""
Update forward transfer metric.
See `Forward Transfer` for more detailed information.
:param k: key to update
:param v: value associated to k
:param initial: update initial value. If False, update
previous value.
"""
super().update(k, v, initial=initial)
def exp_result(self, k=None) -> Union[float, None, Dict[int, float]]:
"""
Result for experience defined by a key.
See `ForwardTransfer` documentation for more detailed information.
k: optional key from which to compute forward transfer.
"""
return super().result(k=k)
def result(self, k=None) -> Union[float, None, Dict[int, float]]:
"""
The average forward transfer over all experiences.
k: optional key from which to compute forward transfer.
"""
return self.stream_forward_transfer.result()
def before_eval(self, strategy) -> None:
super().before_eval(strategy)
self.stream_forward_transfer.reset()
def after_eval_exp(self, strategy: 'BaseStrategy') -> None:
if self.at_init:
self.update(self.eval_exp_id,
self.metric_result(strategy),
initial=True)
else:
if self.train_exp_id == self.eval_exp_id - 1:
self.update(self.eval_exp_id, self.metric_result(strategy))
exp_forward_transfer = self.exp_result(k=self.eval_exp_id)
if exp_forward_transfer is not None:
self.stream_forward_transfer.update(exp_forward_transfer,
weight=1)
def after_eval(self, strategy: 'BaseStrategy') -> \
'MetricResult':
super().after_eval(strategy)
return self._package_result(strategy)
def _package_result(self, strategy: 'BaseStrategy') -> \
MetricResult:
metric_value = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self.get_global_counter()
return [MetricValue(self, metric_name, metric_value, plot_x_position)]
def metric_update(self, strategy):
raise NotImplementedError
def metric_result(self, strategy):
raise NotImplementedError
def __str__(self):
raise NotImplementedError
class GenericStreamForgetting(GenericExperienceForgetting):
"""
The GenericStreamForgetting metric, describing the average evaluation
change in the desired metric detected over all experiences observed
during training.
In particular, the user should override:
* __init__ by calling `super` and instantiating the `self.current_metric`
property as a valid avalanche metric
* `metric_update`, to update `current_metric`
* `metric_result` to get the result from `current_metric`.
* `__str__` to define the experience forgetting name.
This plugin metric, computed over all observed experiences during training,
is the average over the difference between the metric result obtained
after first training on a experience and the metric result obtained
on the same experience at the end of successive experiences.
This metric is computed during the eval phase only.
"""
def __init__(self):
"""
Creates an instance of the GenericStreamForgetting metric.
"""
super().__init__()
self.stream_forgetting = Mean()
"""
The average forgetting over all experiences
"""
def reset(self) -> None:
"""
Resets the forgetting metrics.
Beware that this will also reset the initial metric value of each
experience!
:return: None.
"""
super().reset()
self.stream_forgetting.reset()
def exp_update(self, k, v, initial=False):
"""
Update forgetting metric.
See `Forgetting` for more detailed information.
:param k: key to update
:param v: value associated to k
:param initial: update initial value. If False, update
last value.
"""
super().update(k, v, initial=initial)
def exp_result(self, k=None) -> Union[float, None, Dict[int, float]]:
"""
Result for experience defined by a key.
See `Forgetting` documentation for more detailed information.
k: optional key from which compute forgetting.
"""
return super().result(k)
def result(self, k=None) -> Union[float, None, Dict[int, float]]:
"""
The average forgetting over all experience.
k: optional key from which compute forgetting.
"""
return self.stream_forgetting.result()
def before_eval(self, strategy) -> None:
super().before_eval(strategy)
self.stream_forgetting.reset()
def after_eval_exp(self, strategy: 'BaseStrategy') -> None:
# update experience on which training just ended
if self.train_exp_id == self.eval_exp_id:
self.exp_update(self.eval_exp_id,
self.metric_result(strategy),
initial=True)
else:
# update other experiences
# if experience has not been encountered in training
# its value will not be considered in forgetting
self.exp_update(self.eval_exp_id, self.metric_result(strategy))
# this checks if the evaluation experience has been
# already encountered at training time
# before the last training.
# If not, forgetting should not be returned.
exp_forgetting = self.exp_result(k=self.eval_exp_id)
if exp_forgetting is not None:
self.stream_forgetting.update(exp_forgetting, weight=1)
def after_eval(self, strategy: 'BaseStrategy') -> \
'MetricResult':
return self._package_result(strategy)
def _package_result(self, strategy: 'BaseStrategy') -> \
MetricResult:
metric_value = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self.get_global_counter()
return [MetricValue(self, metric_name, metric_value, plot_x_position)]
def metric_update(self, strategy):
raise NotImplementedError
def metric_result(self, strategy):
raise NotImplementedError
def __str__(self):
raise NotImplementedError
