def test_trainer_can_resume_training_for_exponential_moving_average(self):
moving_average = ExponentialMovingAverage(self.model.named_parameters())
trainer = GradientDescentTrainer(
self.model,
self.optimizer,
self.data_loader,
validation_data_loader=self.validation_data_loader,
num_epochs=1,
serialization_dir=self.TEST_DIR,
moving_average=moving_average,
)
trainer.train()
new_moving_average = ExponentialMovingAverage(self.model.named_parameters())
new_trainer = GradientDescentTrainer(
self.model,
self.optimizer,
self.data_loader,
validation_data_loader=self.validation_data_loader,
num_epochs=3,
serialization_dir=self.TEST_DIR,
moving_average=new_moving_average,
)
epoch = new_trainer._restore_checkpoint()
assert epoch == 1
tracker = trainer._metric_tracker
assert tracker.is_best_so_far()
assert tracker._best_so_far is not None
new_trainer.train()
def test_trainer_can_resume_training_for_exponential_moving_average(self):
moving_average = ExponentialMovingAverage(
self.model.named_parameters())
trainer = Trainer(self.model,
self.optimizer,
self.iterator,
self.instances,
validation_dataset=self.instances,
num_epochs=1,
serialization_dir=self.TEST_DIR,
moving_average=moving_average)
trainer.train()
new_moving_average = ExponentialMovingAverage(
self.model.named_parameters())
new_trainer = Trainer(self.model,
self.optimizer,
self.iterator,
self.instances,
validation_dataset=self.instances,
num_epochs=3,
serialization_dir=self.TEST_DIR,
moving_average=new_moving_average)
epoch = new_trainer._restore_checkpoint() # pylint: disable=protected-access
assert epoch == 1
tracker = trainer._metric_tracker # pylint: disable=protected-access
assert tracker.is_best_so_far()
assert tracker._best_so_far is not None # pylint: disable=protected-access
new_trainer.train()
def test_trainer_can_resume_training_for_exponential_moving_average(self):
moving_average = ExponentialMovingAverage(self.model.named_parameters())
callbacks = self.default_callbacks() + [UpdateMovingAverage(moving_average)]
trainer = CallbackTrainer(self.model,
training_data=self.instances,
iterator=self.iterator,
optimizer=self.optimizer,
num_epochs=1, serialization_dir=self.TEST_DIR,
callbacks=callbacks)
trainer.train()
new_moving_average = ExponentialMovingAverage(self.model.named_parameters())
new_callbacks = self.default_callbacks() + [UpdateMovingAverage(new_moving_average)]
new_trainer = CallbackTrainer(self.model,
training_data=self.instances,
iterator=self.iterator,
optimizer=self.optimizer,
num_epochs=3, serialization_dir=self.TEST_DIR,
callbacks=new_callbacks)
new_trainer.handler.fire_event(Events.TRAINING_START) # pylint: disable=protected-access
assert new_trainer.epoch_number == 1
tracker = trainer.metric_tracker # pylint: disable=protected-access
assert tracker.is_best_so_far()
assert tracker._best_so_far is not None # pylint: disable=protected-access
new_trainer.train()
def test_exponential_moving_average_num_updates(self):
param1 = torch.ones(5, 3)
param2 = torch.ones(2)
moving_average = ExponentialMovingAverage([("param1", param1),
("param2", param2)],
decay=0.9999)
param1.data *= 5 # now all 5s
param2.data *= 10 # now all 10s
moving_average.apply(num_updates=100) # 101 / 110 ~ 0.92 < 0.9999
param1.data *= 5 # now all 25s
param2.data *= 10 # now all 100s
moving_average.apply(
num_updates=1_000_000) # 1_000_001 / 1_000_010 ~ .999991 > .9999
# Get shadow variables
moving_average.assign_average_value()
np.testing.assert_array_almost_equal(
param1,
1 * (101 / 110) * .9999 + 5 * (9 / 110) * .9999 + 25 * 0.0001)
np.testing.assert_array_almost_equal(
param2,
1 * (101 / 110) * .9999 + 10 * (9 / 110) * .9999 + 100 * 0.0001)
# Restore original variables
moving_average.restore()
np.testing.assert_array_almost_equal(param1, 25)
np.testing.assert_array_almost_equal(param2, 100)
def test_exponential_moving_average_without_steps(self):
param1 = torch.ones(5, 3)
param2 = torch.ones(2)
moving_average = ExponentialMovingAverage([("param1", param1),
("param2", param2)],
decay=0.9999)
param1.data *= 5 # now all 5s
param2.data *= 10 # now all 10s
moving_average.apply()
param1.data *= 5 # now all 25s
param2.data *= 10 # now all 100s
moving_average.apply()
# Get shadow variables
moving_average.assign_average_value()
np.testing.assert_array_almost_equal(
param1, 1 * .9999**2 + 5 * .9999 * .0001 + 25 * 0.0001)
np.testing.assert_array_almost_equal(
param2, 1 * .9999**2 + 10 * .9999 * .0001 + 100 * 0.0001)
# Restore original variables
moving_average.restore()
np.testing.assert_array_almost_equal(param1, 25)
np.testing.assert_array_almost_equal(param2, 100)
def test_trainer_can_run_exponential_moving_average(self):
moving_average = ExponentialMovingAverage(self.model.named_parameters(), decay=0.9999)
callbacks = self.default_callbacks() + [ComputeMovingAverage(moving_average)]
trainer = CallbackTrainer(model=self.model,
optimizer=self.optimizer,
num_epochs=2,
callbacks=callbacks)
trainer.train()
def test_works_with_model(self):
class FakeModel(Model):
def __init__(self) -> None:
super().__init__(None)
self.w = torch.nn.Parameter(torch.randn(1))
def forward(
self, t: torch.Tensor
) -> Dict[str, torch.Tensor]: # type: ignore
return {"loss": (t * self.w).sum()}
model = FakeModel()
moving_average = ExponentialMovingAverage(model.named_parameters())
optimizer = torch.optim.SGD(list(model.parameters()), lr=0.1)
for _ in range(10):
optimizer.zero_grad()
t = torch.randn(10)
loss = model.forward(t)["loss"]
loss.backward()
optimizer.step()
moving_average.apply()
w_value = model.w.item()
shadow_value = moving_average._shadows["w"].item()
assert w_value != shadow_value
moving_average.assign_average_value()
assert model.w.item() == shadow_value
moving_average.restore()
assert model.w.item() == w_value
# Now keep training:
for _ in range(10):
optimizer.zero_grad()
t = torch.randn(10)
loss = model.forward(t)["loss"]
loss.backward()
optimizer.step()
moving_average.apply()
def test_trainer_can_run_exponential_moving_average(self):
moving_average = ExponentialMovingAverage(self.model.named_parameters(), decay=0.9999)
trainer = GradientDescentTrainer(
model=self.model,
optimizer=self.optimizer,
data_loader=self.data_loader,
validation_data_loader=self.validation_data_loader,
num_epochs=2,
moving_average=moving_average,
)
trainer.train()
def test_trainer_can_run_exponential_moving_average(self):
moving_average = ExponentialMovingAverage(
self.model.named_parameters(), decay=0.9999)
trainer = Trainer(model=self.model,
optimizer=self.optimizer,
iterator=self.iterator,
train_dataset=self.instances,
validation_dataset=self.instances,
num_epochs=2,
moving_average=moving_average)
trainer.train()
def test_trainer_can_run_exponential_moving_average(self):
moving_average = ExponentialMovingAverage(self.model.named_parameters(), decay=0.9999)
callbacks = self.default_callbacks() + [UpdateMovingAverage(moving_average)]
trainer = CallbackTrainer(
model=self.model,
training_data=self.instances,
iterator=self.iterator,
optimizer=self.optimizer,
num_epochs=2,
callbacks=callbacks,
)
trainer.train()