def find_lr( self, trainer_config: TrainerConfiguration, find_lr_config: FindLRConfiguration, training_data: Union[DataSource, InstancesDataset], ): """Returns a learning rate scan on the model. It increases the learning rate step by step while recording the losses. For a guide on how to select the learning rate please refer to this excellent [blog post](https://towardsdatascience.com/estimating-optimal-learning-rate-for-a-deep-neural-network-ce32f2556ce0) Parameters ---------- trainer_config A trainer configuration find_lr_config A configuration for finding the learning rate training_data The training data Returns ------- (learning_rates, losses) Returns a list of learning rates and corresponding losses. Note: The losses are recorded before applying the corresponding learning rate """ from biome.text._helpers import create_trainer_for_finding_lr # The original pipeline keeps unchanged find_lr_pipeline = self._make_copy() if vocabulary.is_empty(find_lr_pipeline.backbone.vocab, self.config.features.keys): raise EmptyVocabError( "Found an empty vocabulary. " "You probably forgot to create a vocabulary with '.create_vocabulary()'." ) if isinstance(training_data, DataSource): training_data = find_lr_pipeline.create_dataset(training_data) trainer = create_trainer_for_finding_lr( pipeline=find_lr_pipeline, trainer_config=trainer_config, training_data=training_data, ) learning_rates, losses = search_learning_rate( trainer=trainer, start_lr=find_lr_config.start_lr, end_lr=find_lr_config.end_lr, num_batches=find_lr_config.num_batches, linear_steps=find_lr_config.linear_steps, stopping_factor=find_lr_config.stopping_factor, ) return learning_rates, losses
def test_search_learning_rate_without_stopping_factor(self): learning_rates, losses = search_learning_rate(self.trainer, num_batches=100, stopping_factor=None) assert len(learning_rates) == 101 assert len(losses) == 101
def test_search_learning_rate_linear_steps(self): learning_rates_losses = search_learning_rate(self.trainer, linear_steps=True) assert len(learning_rates_losses) > 1
def test_search_learning_rate_with_num_batches_less_than_ten(self): with pytest.raises(ConfigurationError): search_learning_rate(self.trainer, num_batches=9)
def find_lr( self, trainer_config: TrainerConfiguration, find_lr_config: FindLRConfiguration, training_data: Union[Dataset, InstancesDataset], vocab_config: Optional[Union[VocabularyConfiguration, str]] = "default", lazy: bool = False, ): """Returns a learning rate scan on the model. It increases the learning rate step by step while recording the losses. For a guide on how to select the learning rate please refer to this excellent [blog post](https://towardsdatascience.com/estimating-optimal-learning-rate-for-a-deep-neural-network-ce32f2556ce0) Parameters ---------- trainer_config A trainer configuration find_lr_config A configuration for finding the learning rate training_data The training data vocab_config A `VocabularyConfiguration` to create/extend the pipeline's vocabulary if necessary. If 'default' (str), we will use the default configuration `VocabularyConfiguration(datasets=[training_data])`. If None, we will leave the pipeline's vocabulary untouched. lazy If true, dataset instances are lazily loaded from disk, otherwise they are loaded and kept in memory. Returns ------- (learning_rates, losses) Returns a list of learning rates and corresponding losses. Note: The losses are recorded before applying the corresponding learning rate """ from biome.text._helpers import create_trainer_for_finding_lr self._prepare_vocab(vocab_config=vocab_config, training_data=training_data, lazy=lazy) if isinstance(training_data, Dataset): training_data: AllennlpLazyDataset = training_data.to_instances( pipeline=self, lazy=lazy, ) training_data.index_with(self._model.vocab) trainer = create_trainer_for_finding_lr( model=self._model, trainer_config=trainer_config, training_data=training_data, ) # restore the state after the lr search is done tmp_state_path = (Path(tempfile.gettempdir()) / f"model_state_before_findlr_{id(self)}.pth") torch.save(self._model.state_dict(), tmp_state_path) try: learning_rates, losses = search_learning_rate( trainer=trainer, start_lr=find_lr_config.start_lr, end_lr=find_lr_config.end_lr, num_batches=find_lr_config.num_batches, linear_steps=find_lr_config.linear_steps, stopping_factor=find_lr_config.stopping_factor, ) finally: self._model.load_state_dict(torch.load(tmp_state_path)) return learning_rates, losses