def from_params(cls, parameters: List, params: Params) -> 'GanOptimizer': # Because we "tagged" the parameters, we can use getattr to figure out # which ones go with which model. generator_parameters = [("", param) for param in parameters if hasattr(param, '_generator')] discriminator_parameters = [("", param) for param in parameters if hasattr(param, '_discriminator')] generator_optimizer = Optimizer.from_params( generator_parameters, params.pop("generator_optimizer")) discriminator_optimizer = Optimizer.from_params( discriminator_parameters, params.pop("discriminator_optimizer")) return cls(generator_optimizer=generator_optimizer, discriminator_optimizer=discriminator_optimizer)
def test_reduce_on_plateau_error_throw_when_no_metrics_exist(self): with self.assertRaises(ConfigurationError) as context: LearningRateScheduler.from_params(Optimizer.from_params(self.model.named_parameters(), Params({"type": "adam"})), Params({"type": "reduce_on_plateau", "mode": "min"})).step(None, None) assert "learning rate scheduler requires a validation metric" in str(context.exception)
def _get_optimizer(self, lr: float = 1.0): optimizer_params = Params({"type": "sgd", "lr": lr}) optimizer_params["parameter_groups"] = [[[f"^{m}"], {}] for m in self.model._modules] return Optimizer.from_params( model_parameters=self.model.named_parameters(), params=optimizer_params)
def test_can_optimise_model_with_dense_and_sparse_params(self): optimizer_params = Params({"type": "dense_sparse_adam"}) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(model_parameters=parameters, params=optimizer_params) iterator = BasicIterator(2) iterator.index_with(self.vocab) Trainer(self.model, optimizer, iterator, self.instances).train()
def from_params(cls, model: Model, serialization_dir: str, iterator: DataIterator, train_data: Iterable[Instance], validation_data: Optional[Iterable[Instance]], params: Params, validation_iterator: DataIterator = None) -> 'GANTrainer': patience = params.pop_int("patience", None) validation_metric = params.pop("validation_metric", "-loss") shuffle = params.pop_bool("shuffle", True) num_epochs = params.pop_int("num_epochs", 20) cuda_device = params.pop_int("cuda_device", -1) grad_norm = params.pop_float("grad_norm", None) grad_clipping = params.pop_float("grad_clipping", None) lr_scheduler_params = params.pop("learning_rate_scheduler", None) if cuda_device >= 0: model = model.cuda(cuda_device) parameters = [[n, p] for n, p in model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(parameters, params.pop("optimizer")) if lr_scheduler_params: scheduler = LearningRateScheduler.from_params( optimizer, lr_scheduler_params) else: scheduler = None num_serialized_models_to_keep = params.pop_int( "num_serialized_models_to_keep", 20) keep_serialized_model_every_num_seconds = params.pop_int( "keep_serialized_model_every_num_seconds", None) model_save_interval = params.pop_float("model_save_interval", None) summary_interval = params.pop_int("summary_interval", 100) histogram_interval = params.pop_int("histogram_interval", None) params.assert_empty(cls.__name__) return cls(model, optimizer, iterator, train_data, validation_data, patience=patience, validation_metric=validation_metric, validation_iterator=validation_iterator, shuffle=shuffle, num_epochs=num_epochs, serialization_dir=serialization_dir, cuda_device=cuda_device, grad_norm=grad_norm, grad_clipping=grad_clipping, learning_rate_scheduler=scheduler, num_serialized_models_to_keep=num_serialized_models_to_keep, keep_serialized_model_every_num_seconds= keep_serialized_model_every_num_seconds, model_save_interval=model_save_interval, summary_interval=summary_interval, histogram_interval=histogram_interval)
def test_can_optimise_model_with_dense_and_sparse_params(self): optimizer_params = Params({"type": "dense_sparse_adam"}) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(model_parameters=parameters, params=optimizer_params) for instance in self.instances: instance.index_fields(self.vocab) GradientDescentTrainer(self.model, optimizer, SimpleDataLoader(self.instances, 2)).train()
def _setup_training(self, tasks, train_params, optimizer_params, scheduler_params, iterator): # Task bookkeeping task_infos = {task.name: {} for task in tasks} for task in tasks: task_info = task_infos[task.name] tr_generator = iterator(task.train_data, num_epochs=None, cuda_device=self._cuda_device) task_info['n_tr_batches'] = iterator.get_num_batches( task.train_data) task_info['tr_generator'] = tr_generator task_info['loss'] = 0.0 task_info['total_batches_trained'] = 0 task_info['n_batches_since_val'] = 0 task_info['optimizer'] = Optimizer.from_params( train_params, copy.deepcopy(optimizer_params)) task_info['scheduler'] = LearningRateScheduler.from_params( task_info['optimizer'], copy.deepcopy(scheduler_params)) task_info['stopped'] = False task_info['last_log'] = time.time() # Metric bookkeeping all_metrics = [task.val_metric for task in tasks ] + ['micro_accuracy', 'macro_accuracy'] metric_infos = {metric: {'hist': [], 'stopped': False, 'best': (-1, {})} for \ metric in all_metrics} self._task_infos = task_infos self._metric_infos = metric_infos return task_infos, metric_infos
def test_linear_with_warmup_works_properly(self): scheduler = LearningRateScheduler.from_params( optimizer=Optimizer.from_params( model_parameters=self.model.named_parameters(), params=Params({"type": "sgd", "lr": 1.0}), ), params=Params( { "type": "linear_with_warmup", "warmup_steps": 2, "num_epochs": 2, "num_steps_per_epoch": 3, } ), ) optimizer = scheduler.optimizer # Linear warmup for 2 steps. scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 0.5 # 1.0 * 1/2 scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 1.0 # 1.0 * 2/2 # Linear decay for 4 steps. scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 0.75 scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 0.5 scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 0.25 scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 0.0
def predict_json(self, _: JsonDict, cuda_device: int = -1) -> JsonDict: parameter_filename = 'allennlp/seq2seq.json' serialization_dir = 'retrained' subprocess.check_call(['mkdir', '-p', serialization_dir]) params = Params.from_file(parameter_filename) iterator = DataIterator.from_params(params.pop("iterator")) iterator.index_with(self._model.vocab) parameters = [[n, p] for n, p in self._model.named_parameters() if p.requires_grad] trainer_params = params.pop('trainer') optimizer = Optimizer.from_params(parameters, trainer_params.pop("optimizer")) all_datasets = datasets_from_params(params) train_data = all_datasets['train'] trainer = SimpleTrainer(self._model, optimizer, train_data, iterator) interpreter = Interpreter(self._model, self._dataset_reader, trainer) while True: try: interpreter.cmdloop() except Exception as e: print(e) traceback.print_exc() print('Restarting interpreter cmdloop.')
def test_optimizer_parameter_groups(self): optimizer_params = Params({ "type": "sgd", "lr": 1, "momentum": 5, "parameter_groups": [ # the repeated "bias_" checks a corner case # NOT_A_VARIABLE_NAME displays a warning but does not raise an exception [["weight_i", "bias_", "bias_", "NOT_A_VARIABLE_NAME"], {'lr': 2}], [["tag_projection_layer"], {'lr': 3}], ] }) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(parameters, optimizer_params) param_groups = optimizer.param_groups assert len(param_groups) == 3 assert param_groups[0]['lr'] == 2 assert param_groups[1]['lr'] == 3 # base case uses default lr assert param_groups[2]['lr'] == 1 for k in range(3): assert param_groups[k]['momentum'] == 5 # all LSTM parameters except recurrent connections (those with weight_h in name) assert len(param_groups[0]['params']) == 6 # just the projection weight and bias assert len(param_groups[1]['params']) == 2 # the embedding + recurrent connections left in the default group assert len(param_groups[2]['params']) == 3
def test_optimizer_basic(self): optimizer_params = Params({"type": "sgd", "lr": 1}) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(model_parameters=parameters, params=optimizer_params) param_groups = optimizer.param_groups assert len(param_groups) == 1 assert param_groups[0]["lr"] == 1
def test_noam_learning_rate_schedule_does_not_crash(self): model = torch.nn.Sequential(torch.nn.Linear(10, 10)) lrs = LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(), Params({"type": "adam"})), Params({"type": "noam", "model_size": 10, "warmup_steps": 2000})) lrs.step(None) lrs.step_batch(None)
def test_optimizer_parameter_groups(self): optimizer_params = Params({ u"type": u"sgd", u"lr": 1, u"momentum": 5, u"parameter_groups": [ # the repeated "bias_" checks a corner case # NOT_A_VARIABLE_NAME displays a warning but does not raise an exception [[u"weight_i", u"bias_", u"bias_", u"NOT_A_VARIABLE_NAME"], {u'lr': 2}], [[u"tag_projection_layer"], {u'lr': 3}], ] }) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(parameters, optimizer_params) param_groups = optimizer.param_groups assert len(param_groups) == 3 assert param_groups[0][u'lr'] == 2 assert param_groups[1][u'lr'] == 3 # base case uses default lr assert param_groups[2][u'lr'] == 1 for k in range(3): assert param_groups[k][u'momentum'] == 5 # all LSTM parameters except recurrent connections (those with weight_h in name) assert len(param_groups[0][u'params']) == 6 # just the projection weight and bias assert len(param_groups[1][u'params']) == 2 # the embedding + recurrent connections left in the default group assert len(param_groups[2][u'params']) == 3
def test_exponential_works_properly(self): scheduler = LearningRateScheduler.from_params( optimizer=Optimizer.from_params(self.model.named_parameters(), Params({ "type": "sgd", "lr": 1.0 })), params=Params({ "type": "exponential", "gamma": 0.5 }), ) optimizer = scheduler.lr_scheduler.optimizer optimizer.step() # to avoid a pytorch warning # Initial learning rate should be unchanged for first epoch. assert optimizer.param_groups[0]["lr"] == 1.0 # But since the way PyTorch LR schedulers work is a little wonky, # the LR will also be unchanged for the second epoch (epoch id 0). scheduler.step(epoch=0) assert optimizer.param_groups[0]["lr"] == 1.0 # Now the learning rate starts to be updated... scheduler.step(epoch=1) assert optimizer.param_groups[0]["lr"] == 0.5 scheduler.step(epoch=2) assert optimizer.param_groups[0]["lr"] == 0.5**2
def test_noam_learning_rate_schedule_does_not_crash(self): model = torch.nn.Sequential(torch.nn.Linear(10, 10)) lrs = LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(), Params({"type": "adam"})), Params({"type": "noam", "model_size": 10, "warmup_steps": 2000})) lrs.step(None) lrs.step_batch(None)
def _get_optimizer(self, lr: float = 1.0): return Optimizer.from_params( model_parameters=self.model.named_parameters(), params=Params({ "type": "sgd", "lr": lr }))
def test_polynomial_decay_works_properly(self): scheduler = LearningRateScheduler.from_params( optimizer=Optimizer.from_params( model_parameters=self.model.named_parameters(), params=Params({"type": "sgd", "lr": 1.0}), ), params=Params( { "type": "polynomial_decay", "warmup_steps": 2, "num_epochs": 2, "num_steps_per_epoch": 3, "end_learning_rate": 0.1, "power": 2, } ), ) optimizer = scheduler.optimizer # Linear warmup for 2 steps. scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 0.5 # 1.0 * 1/2 scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 1.0 # 1.0 * 2/2 # Polynomial decay for 4 steps. scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 0.60625 # (1.0 - 0.1) * (3/4) ** 2 + 0.1 scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 0.325 # (1.0 - 0.1) * (2/4) ** 2 + 0.1 scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 0.15625 # (1.0 - 0.1) * (1/4) ** 2 + 0.1 scheduler.step_batch() assert optimizer.param_groups[0]["lr"] == 0.1 # (1.0 - 0.1) * (0/4) ** 2 + 0.1
def test_reduce_on_plateau_works_when_metrics_exist(self): LearningRateScheduler.from_params( optimizer=Optimizer.from_params( model_parameters=self.model.named_parameters(), params=Params({"type": "adam"}) ), params=Params({"type": "reduce_on_plateau"}), ).step(10)
def _get_optimizer(self): return Optimizer.from_params( self.model.named_parameters(), Params({ "type": "sgd", "lr": 1.0, "momentum": self.base_momentum }))
def from_params( # type: ignore cls, params: Params, serialization_dir: str, recover: bool = False, cache_directory: str = None, cache_prefix: str = None, ) -> "CallbackTrainer": pieces = TrainerPieces.from_params(params, serialization_dir, recover, cache_directory, cache_prefix) model = pieces.model params = pieces.params validation_iterator = pieces.validation_iterator or pieces.iterator shuffle = params.pop_bool("shuffle", True) num_epochs = params.pop_int("num_epochs", 20) cuda_device = parse_cuda_device(params.pop("cuda_device", -1)) if isinstance(cuda_device, list): model_device = cuda_device[0] else: model_device = cuda_device if model_device >= 0: # Moving model to GPU here so that the optimizer state gets constructed on # the right device. model = model.cuda(model_device) parameters = [[n, p] for n, p in model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(parameters, params.pop("optimizer")) callbacks_params = params.pop("callbacks", []) callbacks: List[Callback] = [ Callback.from_params( params=callback_params, model=model, optimizer=optimizer, instances=pieces.train_dataset, iterator=pieces.iterator, shuffle=shuffle, validation_data=pieces.validation_dataset, validation_iterator=validation_iterator, serialization_dir=serialization_dir, ) for callback_params in callbacks_params ] params.assert_empty(cls.__name__) return cls( model, pieces.train_dataset, pieces.iterator, optimizer, num_epochs=num_epochs, shuffle=shuffle, serialization_dir=serialization_dir, cuda_device=cuda_device, callbacks=callbacks, )
def test_can_optimise_model_with_dense_and_sparse_params(self): optimizer_params = Params({ "type": "dense_sparse_adam" }) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(parameters, optimizer_params) iterator = BasicIterator(2) iterator.index_with(self.vocab) Trainer(self.model, optimizer, iterator, self.instances).train()
def test_no_metric_wrapper_can_support_none_for_metrics(self): lrs = LearningRateScheduler.from_params( Optimizer.from_params(self.model.named_parameters(), Params({"type": "adam"})), Params({ "type": "step", "step_size": 1 })) lrs.step(None, None)
def test_no_metric_wrapper_can_support_none_for_metrics(self): lrs = LearningRateScheduler.from_params( optimizer=Optimizer.from_params( model_parameters=self.model.named_parameters(), params=Params({"type": "adam"}) ), params=Params({"type": "step", "step_size": 1}), ) lrs.lr_scheduler.optimizer.step() # to avoid a pytorch warning lrs.step(None)
def test_noam_learning_rate_schedule_does_not_crash(self): lrs = LearningRateScheduler.from_params( optimizer=Optimizer.from_params( model_parameters=self.model.named_parameters(), params=Params({"type": "adam"}) ), params=Params({"type": "noam", "model_size": 10, "warmup_steps": 2000}), ) lrs.step(None) lrs.step_batch(None)
def test_optimizer_basic(self): optimizer_params = Params({ "type": "sgd", "lr": 1 }) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(parameters, optimizer_params) param_groups = optimizer.param_groups assert len(param_groups) == 1 assert param_groups[0]['lr'] == 1
def test_reduce_on_plateau_error_throw_when_no_metrics_exist(self): model = torch.nn.Sequential(torch.nn.Linear(10, 10)) with self.assertRaises(ConfigurationError) as context: LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(), Params({"type": "adam"})), Params({"type": "reduce_on_plateau"})).step(None, None) self.assertTrue( 'The reduce_on_plateau learning rate scheduler requires a validation metric' in str(context.exception))
def setup_method(self): super().setup_method() self.model = torch.nn.Sequential(torch.nn.Linear(10, 10)) self.optimizer = Optimizer.from_params( model_parameters=self.model.named_parameters(), params=Params({ "type": "sgd", "lr": 1.0 }), )
def test_parameter_type_inference(self): # Should work ok even with lr as a string optimizer_params = Params({"type": "sgd", "lr": "0.1"}) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(model_parameters=parameters, params=optimizer_params) assert optimizer.defaults["lr"] == 0.1
def test_reduce_on_plateau_error_throw_when_no_metrics_exist(self): with pytest.raises( ConfigurationError, match="learning rate scheduler requires a validation metric" ): LearningRateScheduler.from_params( optimizer=Optimizer.from_params( model_parameters=self.model.named_parameters(), params=Params({"type": "adam"}) ), params=Params({"type": "reduce_on_plateau"}), ).step(None)
def test_reduce_on_plateau_error_throw_when_no_metrics_exist(self): model = torch.nn.Sequential(torch.nn.Linear(10, 10)) with self.assertRaises(ConfigurationError) as context: LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(), Params({"type": "adam"})), Params({"type": "reduce_on_plateau"})).step(None, None) self.assertTrue( 'The reduce_on_plateau learning rate scheduler requires a validation metric' in str(context.exception))
def test_no_metric_wrapper_can_support_none_for_metrics(self): model = torch.nn.Sequential(torch.nn.Linear(10, 10)) lrs = LearningRateScheduler.from_params( Optimizer.from_params(model.named_parameters(), Params({"type": "adam"})), Params({ "type": "step", "step_size": 1 })) lrs.step(None, None)
def test_optimizer_parameter_groups(self): optimizer_params = Params({ "type": "sgd", "lr": 1, "momentum": 5, "parameter_groups": [ # the repeated "bias_" checks a corner case # NOT_A_VARIABLE_NAME displays a warning but does not raise an exception [["weight_i", "bias_", "bias_", "NOT_A_VARIABLE_NAME"], { "lr": 2 }], [["tag_projection_layer"], { "lr": 3 }], [["^text_field_embedder.*$"], { "requires_grad": False }], ], }) # Before initializing the optimizer all params in this module will still require grad. assert all([ param.requires_grad for param in self.model.text_field_embedder.parameters() ]) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(model_parameters=parameters, params=optimizer_params) param_groups = optimizer.param_groups # After initializing the optimizer, requires_grad should be false for all params in this module. assert not any([ param.requires_grad for param in self.model.text_field_embedder.parameters() ]) assert len(param_groups) == 3 assert param_groups[0]["lr"] == 2 assert param_groups[1]["lr"] == 3 # base case uses default lr assert param_groups[2]["lr"] == 1 for k in range(3): assert param_groups[k]["momentum"] == 5 # all LSTM parameters except recurrent connections (those with weight_h in name) assert len(param_groups[0]["params"]) == 6 # just the projection weight and bias assert len(param_groups[1]["params"]) == 2 # the recurrent connections left in the default group assert len(param_groups[2]["params"]) == 2
def test_optimizer_params(self): optimizer_params = Params( { "type": "multi", "optimizers": { "default": {"type": "adam", "lr": 1}, "embedder": {"type": "adam", "lr": 2}, "encoder": {"type": "adam", "lr": 3}, }, "parameter_groups": [ [ ["^text_field_embedder"], { "optimizer_name": "embedder", "betas": (0.9, 0.98), "lr": 2, "weight_decay": 0.01, }, ], [["^encoder.*bias"], {"optimizer_name": "encoder", "lr": 0.001}], [["^encoder.*weight"], {"optimizer_name": "encoder", "lr": 0.002}], [["^tag_projection_layer.*.weight$"], {"lr": 5}], ], } ) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(model_parameters=parameters, params=optimizer_params) # When the MultiOptimizer is initialized, `optimizer.param_groups` stores the parameter groups. # These parameter groups are assigned to their own optimizer by the MultiOptimizer. # Check that changes to the parameters in optimizer.param_groups affect the parameters in # optimizer._grouped_optimizers. regex_optimizer_params = set() regex_optimizer_grouped_optimizer_params = set() for param_group in optimizer.param_groups: # Each param_group should have optimizer options visible so they can be used by schedulers. lr = param_group["lr"] assert lr > 0 params = param_group["params"] for param in params: param.data.zero_() regex_optimizer_params.add(id(param)) # Check that the parameters of the sub-optimizers were also changed. for optimizer in optimizer.optimizers.values(): for param_group in optimizer.param_groups: params = param_group["params"] for param in params: regex_optimizer_grouped_optimizer_params.add(id(param)) assert param.sum() == 0, "Param has non-zero values." assert regex_optimizer_params == regex_optimizer_grouped_optimizer_params
def test_parameter_type_inference(self): # Should work ok even with lr as a string optimizer_params = Params({"type": "sgd", "lr": "0.1"}) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(parameters, optimizer_params) assert optimizer.defaults["lr"] == 0.1 # But should crash (in the Pytorch code) if we don't do the type inference optimizer_params = Params({ "type": "sgd", "lr": "0.1", "infer_type_and_cast": False }) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] with pytest.raises(TypeError): optimizer = Optimizer.from_params(parameters, optimizer_params)
def from_params(cls, model: Model, serialization_dir: str, iterator: DataIterator, train_data: Iterable[Instance], validation_data: Optional[Iterable[Instance]], params: Params, validation_iterator: DataIterator = None) -> 'Trainer': patience = params.pop_int("patience", None) validation_metric = params.pop("validation_metric", "-loss") num_epochs = params.pop_int("num_epochs", 20) cuda_device = params.pop_int("cuda_device", -1) grad_norm = params.pop_float("grad_norm", None) grad_clipping = params.pop_float("grad_clipping", None) lr_scheduler_params = params.pop("learning_rate_scheduler", None) if cuda_device >= 0: model = model.cuda(cuda_device) parameters = [[n, p] for n, p in model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(parameters, params.pop("optimizer")) if lr_scheduler_params: scheduler = LearningRateScheduler.from_params(optimizer, lr_scheduler_params) else: scheduler = None num_serialized_models_to_keep = params.pop_int("num_serialized_models_to_keep", 20) keep_serialized_model_every_num_seconds = params.pop_int( "keep_serialized_model_every_num_seconds", None) model_save_interval = params.pop_float("model_save_interval", None) summary_interval = params.pop_int("summary_interval", 100) histogram_interval = params.pop_int("histogram_interval", None) params.assert_empty(cls.__name__) return Trainer(model, optimizer, iterator, train_data, validation_data, patience=patience, validation_metric=validation_metric, validation_iterator=validation_iterator, num_epochs=num_epochs, serialization_dir=serialization_dir, cuda_device=cuda_device, grad_norm=grad_norm, grad_clipping=grad_clipping, learning_rate_scheduler=scheduler, num_serialized_models_to_keep=num_serialized_models_to_keep, keep_serialized_model_every_num_seconds=keep_serialized_model_every_num_seconds, model_save_interval=model_save_interval, summary_interval=summary_interval, histogram_interval=histogram_interval)
def _setup_for_training(self): """Create vocab, configure default loggers/callbacks, create optimizer/lr scheduler, setup best metrics""" # create vocab if self._vocab_config is not None: vocab_datasets = [self._train_instances] if ( self._valid_instances is not None and self._vocab_config.include_valid_data ): vocab_datasets += [self._valid_instances] self._pipeline.create_vocab(vocab_datasets, config=self._vocab_config) # Check for an empty vocab if vocabulary.is_empty( self._pipeline.vocab, self._pipeline.config.features.configured_namespaces ): raise EmptyVocabError( "All your features need a non-empty vocabulary for a training!" ) # we give some special attention to these loggers/callbacks self._wandb_logger: Optional[WandbLogger] = None self._model_checkpoint: Optional[ModelCheckpoint] = None # add default callbacks/loggers self._trainer_config.callbacks = self._add_default_callbacks() if self._trainer_config.logger is not False: self._trainer_config.logger = self._add_default_loggers() # create optimizer, has to come AFTER creating the vocab! self._pipeline.model.optimizer = Optimizer.from_params( Params( { "model_parameters": self._pipeline.model.named_parameters(), **self._trainer_config.optimizer, } ) ) # create lr scheduler, has to come AFTER creating the optimizer! if not ( self._trainer_config.warmup_steps == 0 and self._trainer_config.lr_decay is None ): self._pipeline.model.lr_scheduler = self._create_lr_scheduler() else: self._pipeline.model.lr_scheduler = None # set monitor and mode for best validation metrics self._pipeline.model.monitor = self._trainer_config.monitor self._pipeline.model.monitor_mode = self._trainer_config.monitor_mode
def test_parameter_type_inference(self): # Should work ok even with lr as a string optimizer_params = Params({ "type": "sgd", "lr": "0.1" }) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(parameters, optimizer_params) assert optimizer.defaults["lr"] == 0.1 # But should crash (in the Pytorch code) if we don't do the type inference optimizer_params = Params({ "type": "sgd", "lr": "0.1", "infer_type_and_cast": False }) parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad] with pytest.raises(TypeError): optimizer = Optimizer.from_params(parameters, optimizer_params)
def test_no_metric_wrapper_can_support_none_for_metrics(self): model = torch.nn.Sequential(torch.nn.Linear(10, 10)) lrs = LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(), Params({"type": "adam"})), Params({"type": "step", "step_size": 1})) lrs.step(None, None)
def _get_optimizer(self, lr: float = 1.0): return Optimizer.from_params(self.model.named_parameters(), Params({"type": "sgd", "lr": lr}))
def _get_optimizer(self, lr: float = 1.0): optimizer_params = Params({"type": "sgd", "lr": lr}) optimizer_params["parameter_groups"] = [[[f"^{m}"], {}] for m in self.model._modules] return Optimizer.from_params(self.model.named_parameters(), optimizer_params)
def test_reduce_on_plateau_works_when_metrics_exist(self): model = torch.nn.Sequential(torch.nn.Linear(10, 10)) LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(), Params({"type": "adam"})), Params({"type": "reduce_on_plateau"})).step(10, None)