def make_train_dataloader(self,
num_workers=0,
shuffle=True,
pin_memory=True,
persistent_workers=False,
**kwargs):
"""Data loader initialization.
Called at the start of each learning experience after the dataset
adaptation.
:param num_workers: number of thread workers for the data loading.
:param shuffle: True if the data should be shuffled, False otherwise.
:param pin_memory: If True, the data loader will copy Tensors into CUDA
pinned memory before returning them. Defaults to True.
"""
other_dataloader_args = {}
if parse_version(torch.__version__) >= parse_version('1.7.0'):
other_dataloader_args['persistent_workers'] = persistent_workers
self.dataloader = TaskBalancedDataLoader(self.adapted_dataset,
oversample_small_groups=True,
num_workers=num_workers,
batch_size=self.train_mb_size,
shuffle=shuffle,
pin_memory=pin_memory,
**other_dataloader_args)
def make_train_dataloader(self, shuffle=True, **kwargs):
# you can override make_train_dataloader to change the
# strategy's dataloader
# remember to iterate over self.adapted_dataset
self.dataloader = TaskBalancedDataLoader(
self.adapted_dataset, batch_size=self.train_mb_size
)
def make_train_dataloader(self,
num_workers=0,
shuffle=True,
pin_memory=True,
persistent_workers=False,
**kwargs):
"""Data loader initialization.
Called at the start of each learning experience after the dataset
adaptation.
:param num_workers: number of thread workers for the data loading.
:param shuffle: True if the data should be shuffled, False otherwise.
:param pin_memory: If True, the data loader will copy Tensors into CUDA
pinned memory before returning them. Defaults to True.
:param persistent_workers: If True, the data loader will not shutdown
the worker processes after a dataset has been consumed once.
Used only if `PyTorch >= 1.7.0`.
"""
other_dataloader_args = {}
if parse_version(torch.__version__) >= parse_version("1.7.0"):
other_dataloader_args["persistent_workers"] = persistent_workers
self.dataloader = TaskBalancedDataLoader(
self.adapted_dataset,
oversample_small_groups=True,
num_workers=num_workers,
batch_size=self.train_mb_size,
shuffle=shuffle,
pin_memory=pin_memory,
collate_mbatches=detection_collate_mbatches_fn,
collate_fn=detection_collate_fn,
**other_dataloader_args)
def make_train_dataloader(self,
num_workers=0,
shuffle=True,
pin_memory=True,
**kwargs):
self.dataloader = TaskBalancedDataLoader(self.adapted_dataset,
oversample_small_groups=True,
num_workers=num_workers,
batch_size=self.train_mb_size,
shuffle=shuffle,
pin_memory=pin_memory)
def make_train_dataloader(self, num_workers=0, shuffle=True, **kwargs):
"""
Called after the dataset adaptation. Initializes the data loader.
:param num_workers: number of thread workers for the data loading.
:param shuffle: True if the data should be shuffled, False otherwise.
"""
self.dataloader = TaskBalancedDataLoader(self.adapted_dataset,
oversample_small_groups=True,
num_workers=num_workers,
batch_size=self.train_mb_size,
shuffle=shuffle)
def _verify_rop_tests_reproducibility(self, init_strategy, n_epochs,
criterion):
# This doesn't actually test the support for the specific scheduler
# (ReduceLROnPlateau), but it's only used to check if:
# - the same model+benchmark pair can be instantiated in a
# deterministic way.
# - the same results could be obtained in a standard training loop in a
# deterministic way.
models_rnd = []
benchmarks_rnd = []
for _ in range(2):
benchmark, model = init_strategy()
models_rnd.append(model)
benchmarks_rnd.append(benchmark)
self.assert_model_equals(*models_rnd)
self.assert_benchmark_equals(*benchmarks_rnd)
expected_lrs_rnd = []
for _ in range(2):
benchmark, model = init_strategy()
expected_lrs = []
model.train()
for exp in benchmark.train_stream:
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
expected_lrs.append([])
train_loss = Mean()
for epoch in range(n_epochs):
train_loss.reset()
for x, y, t in TaskBalancedDataLoader(
exp.dataset,
oversample_small_groups=True,
num_workers=0,
batch_size=32,
shuffle=False,
pin_memory=False,
):
optimizer.zero_grad()
outputs = model(x)
loss = criterion(outputs, y)
train_loss.update(loss, weight=len(x))
loss.backward()
optimizer.step()
for group in optimizer.param_groups:
expected_lrs[-1].append(group["lr"])
break
scheduler.step(train_loss.result())
expected_lrs_rnd.append(expected_lrs)
self.assertEqual(expected_lrs_rnd[0], expected_lrs_rnd[1])
def test_basic(self):
benchmark = get_fast_benchmark()
ds = [el.dataset for el in benchmark.train_stream]
data = AvalancheConcatDataset(ds)
dl = TaskBalancedDataLoader(data)
for el in dl:
pass
dl = GroupBalancedDataLoader(ds)
for el in dl:
pass
dl = ReplayDataLoader(data, data)
for el in dl:
pass
def test_basic(self):
scenario = get_fast_scenario()
ds = [el.dataset for el in scenario.train_stream]
data = AvalancheConcatDataset(ds)
dl = TaskBalancedDataLoader(data)
for el in dl:
pass
dl = GroupBalancedDataLoader(ds)
for el in dl:
pass
dl = ReplayDataLoader(data, data)
for el in dl:
pass
def make_train_dataloader(self, num_workers=0, shuffle=True,
pin_memory=True, **kwargs):
"""
Called after the dataset adaptation. Initializes the data loader.
:param num_workers: number of thread workers for the data loading.
:param shuffle: True if the data should be shuffled, False otherwise.
:param pin_memory: If True, the data loader will copy Tensors into CUDA
pinned memory before returning them. Defaults to True.
"""
self.dataloader = TaskBalancedDataLoader(
self.adapted_dataset,
oversample_small_groups=True,
num_workers=num_workers,
batch_size=self.train_mb_size,
shuffle=shuffle,
pin_memory=pin_memory)
def test_dataload_batch_balancing(self):
benchmark = get_fast_benchmark()
batch_size = 32
replayPlugin = ReplayPlugin(mem_size=20)
model = SimpleMLP(input_size=6, hidden_size=10)
cl_strategy = Naive(
model,
SGD(model.parameters(), lr=0.001, momentum=0.9,
weight_decay=0.001),
CrossEntropyLoss(),
train_mb_size=batch_size,
train_epochs=1,
eval_mb_size=100,
plugins=[replayPlugin],
)
for step in benchmark.train_stream:
adapted_dataset = step.dataset
if len(replayPlugin.storage_policy.buffer) > 0:
dataloader = ReplayDataLoader(
adapted_dataset,
replayPlugin.storage_policy.buffer,
oversample_small_tasks=True,
num_workers=0,
batch_size=batch_size,
shuffle=True,
)
else:
dataloader = TaskBalancedDataLoader(adapted_dataset)
for mini_batch in dataloader:
mb_task_labels = mini_batch[-1]
lengths = []
for task_id in adapted_dataset.task_set:
len_task = (mb_task_labels == task_id).sum()
lengths.append(len_task)
if sum(lengths) == batch_size:
difference = max(lengths) - min(lengths)
self.assertLessEqual(difference, 1)
self.assertLessEqual(sum(lengths), batch_size)
cl_strategy.train(step)
if __name__ == "__main__":
benchmark = SplitMNIST(n_experiences=5)
model = SimpleMLP(input_size=784, hidden_size=10)
opt = SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001)
# we use our custom strategy to change the dataloading policy.
cl_strategy = MyCumulativeStrategy(
model,
opt,
CrossEntropyLoss(),
train_epochs=1,
train_mb_size=512,
eval_mb_size=512,
)
for step in benchmark.train_stream:
cl_strategy.train(step)
cl_strategy.eval(step)
# If you don't use avalanche's strategies you can also use the dataloader
# directly to iterate the data
data = step.dataset
dl = TaskBalancedDataLoader(data)
for x, y, t in dl:
# by default minibatches in Avalanche have the form <x, y, ..., t>
# with arbitrary additional tensors between y and t.
print(x, y, t)
break
def test_scheduler_reduce_on_plateau_plugin_with_val_stream(self):
# Regression test for issue #858 (part 2)
n_epochs = 20
criterion = CrossEntropyLoss()
def _prepare_rng_critical_parts(seed=1234):
torch.random.manual_seed(seed)
initial_benchmark = PluginTests.create_benchmark(seed=seed)
val_benchmark = benchmark_with_validation_stream(
initial_benchmark, 0.3, shuffle=True
)
return (val_benchmark, _PlainMLP(input_size=6, hidden_size=10))
self._verify_rop_tests_reproducibility(
_prepare_rng_critical_parts, n_epochs, criterion
)
# Everything is in order, now we can test the plugin support for the
# ReduceLROnPlateau scheduler!
for reset_lr, reset_scheduler in itertools.product(
(True, False), (True, False)
):
with self.subTest(
reset_lr=reset_lr, reset_scheduler=reset_scheduler
):
# First, obtain the reference (expected) lr timeline by running
# a plain PyTorch training loop with ReduceLROnPlateau.
benchmark, model = _prepare_rng_critical_parts()
expected_lrs = []
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
for exp_idx, exp in enumerate(benchmark.train_stream):
expected_lrs.append([])
model.train()
if reset_lr:
for group in optimizer.param_groups:
group["lr"] = 0.001
if reset_scheduler:
scheduler = ReduceLROnPlateau(optimizer)
for epoch in range(n_epochs):
for x, y, t in TaskBalancedDataLoader(
exp.dataset,
oversample_small_groups=True,
num_workers=0,
batch_size=32,
shuffle=False,
pin_memory=False,
):
optimizer.zero_grad()
outputs = model(x)
loss = criterion(outputs, y)
loss.backward()
optimizer.step()
for group in optimizer.param_groups:
expected_lrs[-1].append(group["lr"])
break
val_loss = Mean()
val_exp = benchmark.valid_stream[exp_idx]
model.eval()
with torch.no_grad():
for x, y, t in DataLoader(
val_exp.dataset,
num_workers=0,
batch_size=100,
pin_memory=False,
):
outputs = model(x)
loss = criterion(outputs, y)
val_loss.update(loss, weight=len(x))
scheduler.step(val_loss.result())
# Now we have the correct timeline stored in expected_lrs
# Let's test the plugin!
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
PluginTests._test_scheduler_plugin(
benchmark,
model,
optimizer,
scheduler,
n_epochs,
reset_lr,
reset_scheduler,
expected_lrs,
criterion=criterion,
metric="val_loss",
eval_on_valid_stream=True,
)
def test_scheduler_reduce_on_plateau_plugin(self):
# Regression test for issue #858
n_epochs = 20
criterion = CrossEntropyLoss()
def _prepare_rng_critical_parts(seed=1234):
torch.random.manual_seed(seed)
return (
PluginTests.create_benchmark(seed=seed),
_PlainMLP(input_size=6, hidden_size=10),
)
self._verify_rop_tests_reproducibility(
_prepare_rng_critical_parts, n_epochs, criterion
)
# Everything is in order, now we can test the plugin support for the
# ReduceLROnPlateau scheduler!
for reset_lr, reset_scheduler in itertools.product(
(True, False), (True, False)
):
with self.subTest(
reset_lr=reset_lr, reset_scheduler=reset_scheduler
):
# First, obtain the reference (expected) lr timeline by running
# a plain PyTorch training loop with ReduceLROnPlateau.
benchmark, model = _prepare_rng_critical_parts()
model.train()
expected_lrs = []
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
for exp in benchmark.train_stream:
if reset_lr:
for group in optimizer.param_groups:
group["lr"] = 0.001
if reset_scheduler:
scheduler = ReduceLROnPlateau(optimizer)
expected_lrs.append([])
train_loss = Mean()
for epoch in range(n_epochs):
train_loss.reset()
for x, y, t in TaskBalancedDataLoader(
exp.dataset,
oversample_small_groups=True,
num_workers=0,
batch_size=32,
shuffle=False,
pin_memory=False,
):
optimizer.zero_grad()
outputs = model(x)
loss = criterion(outputs, y)
train_loss.update(loss, weight=len(x))
loss.backward()
optimizer.step()
scheduler.step(train_loss.result())
for group in optimizer.param_groups:
expected_lrs[-1].append(group["lr"])
break
# Now we have the correct timeline stored in expected_lrs.
# Let's test the plugin!
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
PluginTests._test_scheduler_plugin(
benchmark,
model,
optimizer,
scheduler,
n_epochs,
reset_lr,
reset_scheduler,
expected_lrs,
criterion=criterion,
metric="train_loss",
)
# Other tests
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
scheduler2 = MultiStepLR(optimizer, [1, 2, 3])
# The metric must be set
with self.assertRaises(Exception):
LRSchedulerPlugin(scheduler, metric=None)
# Doesn't make sense to set the metric when using a non-metric
# based scheduler (should warn)
with self.assertWarns(Warning):
LRSchedulerPlugin(scheduler2, metric="train_loss")
# Must raise an error on unsupported metric
with self.assertRaises(Exception):
LRSchedulerPlugin(scheduler, metric="cuteness")