def find_lr(
self,
trainer_config: TrainerConfiguration,
find_lr_config: FindLRConfiguration,
training_data: Union[DataSource, InstancesDataset],
):
"""Returns a learning rate scan on the model.
It increases the learning rate step by step while recording the losses.
For a guide on how to select the learning rate please refer to this excellent
[blog post](https://towardsdatascience.com/estimating-optimal-learning-rate-for-a-deep-neural-network-ce32f2556ce0)
Parameters
----------
trainer_config
A trainer configuration
find_lr_config
A configuration for finding the learning rate
training_data
The training data
Returns
-------
(learning_rates, losses)
Returns a list of learning rates and corresponding losses.
Note: The losses are recorded before applying the corresponding learning rate
"""
from biome.text._helpers import create_trainer_for_finding_lr
# The original pipeline keeps unchanged
find_lr_pipeline = self._make_copy()
if vocabulary.is_empty(find_lr_pipeline.backbone.vocab,
self.config.features.keys):
raise EmptyVocabError(
"Found an empty vocabulary. "
"You probably forgot to create a vocabulary with '.create_vocabulary()'."
)
if isinstance(training_data, DataSource):
training_data = find_lr_pipeline.create_dataset(training_data)
trainer = create_trainer_for_finding_lr(
pipeline=find_lr_pipeline,
trainer_config=trainer_config,
training_data=training_data,
)
learning_rates, losses = search_learning_rate(
trainer=trainer,
start_lr=find_lr_config.start_lr,
end_lr=find_lr_config.end_lr,
num_batches=find_lr_config.num_batches,
linear_steps=find_lr_config.linear_steps,
stopping_factor=find_lr_config.stopping_factor,
)
return learning_rates, losses
def test_search_learning_rate_without_stopping_factor(self):
learning_rates, losses = search_learning_rate(self.trainer,
num_batches=100,
stopping_factor=None)
assert len(learning_rates) == 101
assert len(losses) == 101
def test_search_learning_rate_linear_steps(self):
learning_rates_losses = search_learning_rate(self.trainer,
linear_steps=True)
assert len(learning_rates_losses) > 1
def test_search_learning_rate_with_num_batches_less_than_ten(self):
with pytest.raises(ConfigurationError):
search_learning_rate(self.trainer, num_batches=9)
def test_search_learning_rate_without_stopping_factor(self):
learning_rates, losses = search_learning_rate(self.trainer, num_batches=100,
stopping_factor=None)
assert len(learning_rates) == 101
assert len(losses) == 101
def test_search_learning_rate_linear_steps(self):
learning_rates_losses = search_learning_rate(self.trainer, linear_steps=True)
assert len(learning_rates_losses) > 1
def test_search_learning_rate_with_num_batches_less_than_ten(self):
with pytest.raises(ConfigurationError):
search_learning_rate(self.trainer, num_batches=9)
def find_lr(
self,
trainer_config: TrainerConfiguration,
find_lr_config: FindLRConfiguration,
training_data: Union[Dataset, InstancesDataset],
vocab_config: Optional[Union[VocabularyConfiguration,
str]] = "default",
lazy: bool = False,
):
"""Returns a learning rate scan on the model.
It increases the learning rate step by step while recording the losses.
For a guide on how to select the learning rate please refer to this excellent
[blog post](https://towardsdatascience.com/estimating-optimal-learning-rate-for-a-deep-neural-network-ce32f2556ce0)
Parameters
----------
trainer_config
A trainer configuration
find_lr_config
A configuration for finding the learning rate
training_data
The training data
vocab_config
A `VocabularyConfiguration` to create/extend the pipeline's vocabulary if necessary.
If 'default' (str), we will use the default configuration
`VocabularyConfiguration(datasets=[training_data])`.
If None, we will leave the pipeline's vocabulary untouched.
lazy
If true, dataset instances are lazily loaded from disk, otherwise they are loaded and kept in memory.
Returns
-------
(learning_rates, losses)
Returns a list of learning rates and corresponding losses.
Note: The losses are recorded before applying the corresponding learning rate
"""
from biome.text._helpers import create_trainer_for_finding_lr
self._prepare_vocab(vocab_config=vocab_config,
training_data=training_data,
lazy=lazy)
if isinstance(training_data, Dataset):
training_data: AllennlpLazyDataset = training_data.to_instances(
pipeline=self,
lazy=lazy,
)
training_data.index_with(self._model.vocab)
trainer = create_trainer_for_finding_lr(
model=self._model,
trainer_config=trainer_config,
training_data=training_data,
)
# restore the state after the lr search is done
tmp_state_path = (Path(tempfile.gettempdir()) /
f"model_state_before_findlr_{id(self)}.pth")
torch.save(self._model.state_dict(), tmp_state_path)
try:
learning_rates, losses = search_learning_rate(
trainer=trainer,
start_lr=find_lr_config.start_lr,
end_lr=find_lr_config.end_lr,
num_batches=find_lr_config.num_batches,
linear_steps=find_lr_config.linear_steps,
stopping_factor=find_lr_config.stopping_factor,
)
finally:
self._model.load_state_dict(torch.load(tmp_state_path))
return learning_rates, losses
