def test_trainer_can_resume_training_for_exponential_moving_average(self): moving_average = ExponentialMovingAverage(self.model.named_parameters()) trainer = GradientDescentTrainer( self.model, self.optimizer, self.data_loader, validation_data_loader=self.validation_data_loader, num_epochs=1, serialization_dir=self.TEST_DIR, moving_average=moving_average, ) trainer.train() new_moving_average = ExponentialMovingAverage(self.model.named_parameters()) new_trainer = GradientDescentTrainer( self.model, self.optimizer, self.data_loader, validation_data_loader=self.validation_data_loader, num_epochs=3, serialization_dir=self.TEST_DIR, moving_average=new_moving_average, ) epoch = new_trainer._restore_checkpoint() assert epoch == 1 tracker = trainer._metric_tracker assert tracker.is_best_so_far() assert tracker._best_so_far is not None new_trainer.train()
def test_trainer_can_resume_training_for_exponential_moving_average(self): moving_average = ExponentialMovingAverage( self.model.named_parameters()) trainer = Trainer(self.model, self.optimizer, self.iterator, self.instances, validation_dataset=self.instances, num_epochs=1, serialization_dir=self.TEST_DIR, moving_average=moving_average) trainer.train() new_moving_average = ExponentialMovingAverage( self.model.named_parameters()) new_trainer = Trainer(self.model, self.optimizer, self.iterator, self.instances, validation_dataset=self.instances, num_epochs=3, serialization_dir=self.TEST_DIR, moving_average=new_moving_average) epoch = new_trainer._restore_checkpoint() # pylint: disable=protected-access assert epoch == 1 tracker = trainer._metric_tracker # pylint: disable=protected-access assert tracker.is_best_so_far() assert tracker._best_so_far is not None # pylint: disable=protected-access new_trainer.train()
def test_trainer_can_resume_training_for_exponential_moving_average(self): moving_average = ExponentialMovingAverage(self.model.named_parameters()) callbacks = self.default_callbacks() + [UpdateMovingAverage(moving_average)] trainer = CallbackTrainer(self.model, training_data=self.instances, iterator=self.iterator, optimizer=self.optimizer, num_epochs=1, serialization_dir=self.TEST_DIR, callbacks=callbacks) trainer.train() new_moving_average = ExponentialMovingAverage(self.model.named_parameters()) new_callbacks = self.default_callbacks() + [UpdateMovingAverage(new_moving_average)] new_trainer = CallbackTrainer(self.model, training_data=self.instances, iterator=self.iterator, optimizer=self.optimizer, num_epochs=3, serialization_dir=self.TEST_DIR, callbacks=new_callbacks) new_trainer.handler.fire_event(Events.TRAINING_START) # pylint: disable=protected-access assert new_trainer.epoch_number == 1 tracker = trainer.metric_tracker # pylint: disable=protected-access assert tracker.is_best_so_far() assert tracker._best_so_far is not None # pylint: disable=protected-access new_trainer.train()
def test_exponential_moving_average_num_updates(self): param1 = torch.ones(5, 3) param2 = torch.ones(2) moving_average = ExponentialMovingAverage([("param1", param1), ("param2", param2)], decay=0.9999) param1.data *= 5 # now all 5s param2.data *= 10 # now all 10s moving_average.apply(num_updates=100) # 101 / 110 ~ 0.92 < 0.9999 param1.data *= 5 # now all 25s param2.data *= 10 # now all 100s moving_average.apply( num_updates=1_000_000) # 1_000_001 / 1_000_010 ~ .999991 > .9999 # Get shadow variables moving_average.assign_average_value() np.testing.assert_array_almost_equal( param1, 1 * (101 / 110) * .9999 + 5 * (9 / 110) * .9999 + 25 * 0.0001) np.testing.assert_array_almost_equal( param2, 1 * (101 / 110) * .9999 + 10 * (9 / 110) * .9999 + 100 * 0.0001) # Restore original variables moving_average.restore() np.testing.assert_array_almost_equal(param1, 25) np.testing.assert_array_almost_equal(param2, 100)
def test_exponential_moving_average_without_steps(self): param1 = torch.ones(5, 3) param2 = torch.ones(2) moving_average = ExponentialMovingAverage([("param1", param1), ("param2", param2)], decay=0.9999) param1.data *= 5 # now all 5s param2.data *= 10 # now all 10s moving_average.apply() param1.data *= 5 # now all 25s param2.data *= 10 # now all 100s moving_average.apply() # Get shadow variables moving_average.assign_average_value() np.testing.assert_array_almost_equal( param1, 1 * .9999**2 + 5 * .9999 * .0001 + 25 * 0.0001) np.testing.assert_array_almost_equal( param2, 1 * .9999**2 + 10 * .9999 * .0001 + 100 * 0.0001) # Restore original variables moving_average.restore() np.testing.assert_array_almost_equal(param1, 25) np.testing.assert_array_almost_equal(param2, 100)
def test_trainer_can_run_exponential_moving_average(self): moving_average = ExponentialMovingAverage(self.model.named_parameters(), decay=0.9999) callbacks = self.default_callbacks() + [ComputeMovingAverage(moving_average)] trainer = CallbackTrainer(model=self.model, optimizer=self.optimizer, num_epochs=2, callbacks=callbacks) trainer.train()
def test_trainer_can_run_exponential_moving_average(self): moving_average = ExponentialMovingAverage(self.model.named_parameters(), decay=0.9999) trainer = GradientDescentTrainer( model=self.model, optimizer=self.optimizer, data_loader=self.data_loader, validation_data_loader=self.validation_data_loader, num_epochs=2, moving_average=moving_average, ) trainer.train()
def test_trainer_can_run_exponential_moving_average(self): moving_average = ExponentialMovingAverage( self.model.named_parameters(), decay=0.9999) trainer = Trainer(model=self.model, optimizer=self.optimizer, iterator=self.iterator, train_dataset=self.instances, validation_dataset=self.instances, num_epochs=2, moving_average=moving_average) trainer.train()
def test_trainer_can_run_exponential_moving_average(self): moving_average = ExponentialMovingAverage(self.model.named_parameters(), decay=0.9999) callbacks = self.default_callbacks() + [UpdateMovingAverage(moving_average)] trainer = CallbackTrainer( model=self.model, training_data=self.instances, iterator=self.iterator, optimizer=self.optimizer, num_epochs=2, callbacks=callbacks, ) trainer.train()
def test_works_with_model(self): class FakeModel(Model): def __init__(self) -> None: super().__init__(None) self.w = torch.nn.Parameter(torch.randn(1)) def forward( self, t: torch.Tensor ) -> Dict[str, torch.Tensor]: # type: ignore return {"loss": (t * self.w).sum()} model = FakeModel() moving_average = ExponentialMovingAverage(model.named_parameters()) optimizer = torch.optim.SGD(list(model.parameters()), lr=0.1) for _ in range(10): optimizer.zero_grad() t = torch.randn(10) loss = model.forward(t)["loss"] loss.backward() optimizer.step() moving_average.apply() w_value = model.w.item() shadow_value = moving_average._shadows["w"].item() assert w_value != shadow_value moving_average.assign_average_value() assert model.w.item() == shadow_value moving_average.restore() assert model.w.item() == w_value # Now keep training: for _ in range(10): optimizer.zero_grad() t = torch.randn(10) loss = model.forward(t)["loss"] loss.backward() optimizer.step() moving_average.apply()