def test_noam_learning_rate_schedule_does_not_crash(self):
model = torch.nn.Sequential(torch.nn.Linear(10, 10))
lrs = LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(),
Params({"type": "adam"})),
Params({"type": "noam", "model_size": 10, "warmup_steps": 2000}))
lrs.step(None)
lrs.step_batch(None)
def test_optimizer_parameter_groups(self):
optimizer_params = Params({
"type": "sgd",
"lr": 1,
"momentum": 5,
"parameter_groups": [
# the repeated "bias_" checks a corner case
# NOT_A_VARIABLE_NAME displays a warning but does not raise an exception
[["weight_i", "bias_", "bias_", "NOT_A_VARIABLE_NAME"], {'lr': 2}],
[["tag_projection_layer"], {'lr': 3}],
]
})
parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad]
optimizer = Optimizer.from_params(parameters, optimizer_params)
param_groups = optimizer.param_groups
assert len(param_groups) == 3
assert param_groups[0]['lr'] == 2
assert param_groups[1]['lr'] == 3
# base case uses default lr
assert param_groups[2]['lr'] == 1
for k in range(3):
assert param_groups[k]['momentum'] == 5
# all LSTM parameters except recurrent connections (those with weight_h in name)
assert len(param_groups[0]['params']) == 6
# just the projection weight and bias
assert len(param_groups[1]['params']) == 2
# the embedding + recurrent connections left in the default group
assert len(param_groups[2]['params']) == 3
def test_can_optimise_model_with_dense_and_sparse_params(self):
optimizer_params = Params({
"type": "dense_sparse_adam"
})
parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad]
optimizer = Optimizer.from_params(parameters, optimizer_params)
iterator = BasicIterator(2)
iterator.index_with(self.vocab)
Trainer(self.model, optimizer, iterator, self.instances).train()
def test_reduce_on_plateau_error_throw_when_no_metrics_exist(self):
model = torch.nn.Sequential(torch.nn.Linear(10, 10))
with self.assertRaises(ConfigurationError) as context:
LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(),
Params({"type": "adam"})),
Params({"type": "reduce_on_plateau"})).step(None, None)
self.assertTrue(
'The reduce_on_plateau learning rate scheduler requires a validation metric'
in str(context.exception))
def test_optimizer_basic(self):
optimizer_params = Params({
"type": "sgd",
"lr": 1
})
parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad]
optimizer = Optimizer.from_params(parameters, optimizer_params)
param_groups = optimizer.param_groups
assert len(param_groups) == 1
assert param_groups[0]['lr'] == 1
def from_params(cls,
model: Model,
serialization_dir: str,
iterator: DataIterator,
train_data: Iterable[Instance],
validation_data: Optional[Iterable[Instance]],
params: Params,
validation_iterator: DataIterator = None) -> 'Trainer':
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = params.pop_int("cuda_device", -1)
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
if cuda_device >= 0:
model = model.cuda(cuda_device)
parameters = [[n, p] for n, p in model.named_parameters() if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if lr_scheduler_params:
scheduler = LearningRateScheduler.from_params(optimizer, lr_scheduler_params)
else:
scheduler = None
num_serialized_models_to_keep = params.pop_int("num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
params.assert_empty(cls.__name__)
return Trainer(model, optimizer, iterator,
train_data, validation_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=scheduler,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=keep_serialized_model_every_num_seconds,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval)
def test_parameter_type_inference(self):
# Should work ok even with lr as a string
optimizer_params = Params({
"type": "sgd",
"lr": "0.1"
})
parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad]
optimizer = Optimizer.from_params(parameters, optimizer_params)
assert optimizer.defaults["lr"] == 0.1
# But should crash (in the Pytorch code) if we don't do the type inference
optimizer_params = Params({
"type": "sgd",
"lr": "0.1",
"infer_type_and_cast": False
})
parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad]
with pytest.raises(TypeError):
optimizer = Optimizer.from_params(parameters, optimizer_params)
def from_params(cls,
params: Params,
serialization_dir: str,
recover: bool = False,
cache_directory: str = None,
cache_prefix: str = None) -> 'PtTrainer':
max_src_len = params.dataset_reader.get('max_src_len', None)
all_datasets = training_util.datasets_from_params(
params, cache_directory, cache_prefix)
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info(
"From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation))
if recover and os.path.exists(
os.path.join(serialization_dir, "vocabulary")):
vocab = Vocabulary.from_files(
os.path.join(serialization_dir, "vocabulary"))
params.pop("vocabulary", {})
else:
vocab = Vocabulary.from_params(params.pop(
"vocabulary", {}), (instance
for key, dataset in all_datasets.items()
if key in datasets_for_vocab_creation
for instance in dataset))
model = Model.from_params(vocab=vocab, params=params.pop('model'))
# If vocab extension is ON for training, embedding extension should also be
# done. If vocab and embeddings are already in sync, it would be a no-op.
model.extend_embedder_vocab()
# Initializing the model can have side effect of expanding the vocabulary
vocab.save_to_files(os.path.join(serialization_dir, "vocabulary"))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(model.vocab)
validation_iterator_params = params.pop("validation_iterator", None)
if validation_iterator_params:
validation_iterator = DataIterator.from_params(
validation_iterator_params)
validation_iterator.index_with(model.vocab)
else:
validation_iterator = None
train_data = all_datasets['train']
validation_data = all_datasets.get('validation')
test_data = all_datasets.get('test')
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
frozen_parameter_names, tunable_parameter_names = \
get_frozen_and_tunable_parameter_names(model)
logger.info("Following parameters are Frozen (without gradient):")
for name in frozen_parameter_names:
logger.info(name)
logger.info("Following parameters are Tunable (with gradient):")
for name in tunable_parameter_names:
logger.info(name)
params = trainer_params
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
momentum_scheduler_params = params.pop("momentum_scheduler", None)
if isinstance(cuda_device, list):
model_device = cuda_device[0]
else:
model_device = cuda_device
if model_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(model_device)
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if "moving_average" in params:
moving_average = MovingAverage.from_params(
params.pop("moving_average"), parameters=parameters)
else:
moving_average = None
if lr_scheduler_params:
lr_scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params)
else:
lr_scheduler = None
if momentum_scheduler_params:
momentum_scheduler = MomentumScheduler.from_params(
optimizer, momentum_scheduler_params)
else:
momentum_scheduler = None
if 'checkpointer' in params:
if 'keep_serialized_model_every_num_seconds' in params or \
'num_serialized_models_to_keep' in params:
raise ConfigurationError(
"Checkpointer may be initialized either from the 'checkpointer' key or from the "
"keys 'num_serialized_models_to_keep' and 'keep_serialized_model_every_num_seconds'"
" but the passed config uses both methods.")
checkpointer = Checkpointer.from_params(params.pop("checkpointer"))
else:
num_serialized_models_to_keep = params.pop_int(
"num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
checkpointer = Checkpointer(
serialization_dir=serialization_dir,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=
keep_serialized_model_every_num_seconds)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool(
"should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate",
False)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
return cls(
model,
optimizer,
iterator,
train_data,
validation_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
max_src_len=max_src_len,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=lr_scheduler,
momentum_scheduler=momentum_scheduler,
checkpointer=checkpointer,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period,
moving_average=moving_average,
batch_size=iterator._batch_size)
def test_reduce_on_plateau_works_when_metrics_exist(self):
model = torch.nn.Sequential(torch.nn.Linear(10, 10))
LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(),
Params({"type": "adam"})),
Params({"type": "reduce_on_plateau"})).step(10, None)
def __init__(self,
model: Model,
task_list: List[Task],
optimizer_params: Params,
lr_scheduler_params: Params,
patience: Optional[int] = None,
num_epochs: int = 20,
serialization_dir: Optional[str] = None,
cuda_device: int = -1,
gradient_accumulation_steps: int = 1,
grad_norm: Optional[float] = None,
grad_clipping: Optional[float] = None,
min_lr: float = 0.00001,
no_tqdm: bool = False,
momentum_scheduler: Optional[MomentumScheduler] = None,
summary_interval: int = 50,
histogram_interval: int = 50,
should_log_parameter_statistics: bool = True,
should_log_learning_rate: bool = True,
sampling_method: str = "proportional",
moving_average: Optional[MovingAverage] = None) -> None:
self._model = model
self._task_list = task_list
self._n_tasks = len(self._task_list)
self._optimizers = {}
self._all_params = [(n, p) for n, p in self._model.named_parameters()
if p.requires_grad]
self._params_share_encoder = [
(n, p) for n, p in self._model.named_parameters()
if p.requires_grad and "_shared_encoder" in n
]
self._params_share_encoder_de = [
(n, p) for n, p in self._model.named_parameters()
if p.requires_grad and
("_shared_encoder" in n or "_seq_vec" in n or "_domain_embeddings"
in n or "_de_attention" in n or "_de_feedforward" in n)
]
self._params_share_discriminator = [
(n, p) for n, p in self._model.named_parameters()
if p.requires_grad and "_s_domain_discriminator" in n
]
self._params_valid_discriminator = [
(n, p) for n, p in self._model.named_parameters()
if p.requires_grad and "_valid_discriminator" in n
]
self._optimizers = dict()
self._optimizers["all_params"] = Optimizer.from_params(
model_parameters=self._all_params,
params=deepcopy(optimizer_params))
self._optimizers["share_encoder"] = Optimizer.from_params(
model_parameters=self._params_share_encoder,
params=deepcopy(optimizer_params))
self._optimizers["share_encoder_de"] = Optimizer.from_params(
model_parameters=self._params_share_encoder_de,
params=deepcopy(optimizer_params))
self._optimizers["share_discriminator"] = Optimizer.from_params(
model_parameters=self._params_share_discriminator,
params=deepcopy(optimizer_params))
self._optimizers["valid_discriminator"] = Optimizer.from_params(
model_parameters=self._params_valid_discriminator,
params=deepcopy(optimizer_params))
self._schedulers = dict()
self._schedulers["all_params"] = LearningRateScheduler.from_params(
optimizer=self._optimizers["all_params"],
params=deepcopy(lr_scheduler_params))
self._schedulers["share_encoder"] = LearningRateScheduler.from_params(
optimizer=self._optimizers["share_encoder"],
params=deepcopy(lr_scheduler_params))
self._schedulers[
"share_encoder_de"] = LearningRateScheduler.from_params(
optimizer=self._optimizers["share_encoder_de"],
params=deepcopy(lr_scheduler_params))
self._schedulers[
"share_discriminator"] = LearningRateScheduler.from_params(
optimizer=self._optimizers["share_discriminator"],
params=deepcopy(lr_scheduler_params))
self._schedulers[
"valid_discriminator"] = LearningRateScheduler.from_params(
optimizer=self._optimizers["valid_discriminator"],
params=deepcopy(lr_scheduler_params))
self._serialization_dir = serialization_dir
self._cuda_device = cuda_device
if self._cuda_device >= 0:
check_for_gpu(self._cuda_device)
self._model = self._model.cuda(self._cuda_device)
self._patience = patience
self._num_epochs = num_epochs
self._epoch_trained = 0
self._gradient_accumulation_steps = gradient_accumulation_steps
self._grad_norm = grad_norm
self._grad_clipping = grad_clipping
self._min_lr = min_lr
self._no_tqdm = no_tqdm
self._sampling_method = sampling_method
self._task_infos = None
self._metric_infos = None
self._tr_generators = None
self._global_step = 0
self._batch_num_total = 0
self._tensorboard = TensorboardWriter(
get_batch_num_total=lambda: self._batch_num_total,
serialization_dir=serialization_dir,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate)
self._last_log = 0.0 # time of last logging
# Enable activation logging.
if histogram_interval is not None:
self._tensorboard.enable_activation_logging(self._model)
def test_reduce_on_plateau_works_when_metrics_exist(self):
LearningRateScheduler.from_params(
Optimizer.from_params(self.model.named_parameters(),
Params({"type": "adam"})),
Params({"type": "reduce_on_plateau"})).step(10, None)
def test_can_optimise_model_with_dense_and_sparse_params(self):
optimizer_params = Params({"type": "dense_sparse_adam"})
parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad]
optimizer = Optimizer.from_params(model_parameters=parameters, params=optimizer_params)
self.instances.index_with(self.vocab)
GradientDescentTrainer(self.model, optimizer, DataLoader(self.instances, 2)).train()
def __init__(
self,
model: Model,
task_list: List[Task],
optimizer_params: Params,
lr_scheduler_params: Params,
patience: Optional[int] = None,
num_epochs: int = 20,
serialization_dir: str = None,
cuda_device: int = -1,
gradient_accumulation_steps: int = 1,
grad_norm: Optional[float] = None,
grad_clipping: Optional[float] = None,
min_lr: float = 0.00001,
no_tqdm: bool = False,
summary_interval: int = 10,
histogram_interval: int = 10,
log_parameter_statistics: bool = False,
log_gradient_statistics: bool = False,
):
"""
Parameters
----------
model: ``Model``, required.
An AllenNLP model to be optimized. Pytorch Modules can also be optimized if
their ``forward`` method returns a dictionary with a "loss" key, containing a
scalar tensor representing the loss function to be optimized.
iterator: ``DataIterator``, required.
A method for iterating over a ``Dataset``, yielding padded indexed batches.
patience: Optional[int] > 0, optional (default=None)
Number of epochs to be patient before early stopping: the training is stopped
after ``patience`` epochs with no improvement. If given, it must be ``> 0``.
If None, early stopping is disabled.
num_epochs: int, optional (default = 20)
Number of training epochs.
serialization_dir: str, optional (default=None)
Path to directory for saving and loading model files. Models will not be saved if
this parameter is not passed.
cuda_device: int, optional (default = -1)
An integer specifying the CUDA device to use. If -1, the CPU is used.
Multi-gpu training is not currently supported, but will be once the
Pytorch DataParallel API stabilises.
grad_norm: float, optional, (default = None).
If provided, gradient norms will be rescaled to have a maximum of this value.
grad_clipping : float, optional (default = None).
If provided, gradients will be clipped `during the backward pass` to have an (absolute)
maximum of this value. If you are getting ``NaNs`` in your gradients during training
that are not solved by using ``grad_norm``, you may need this.
no_tqdm : bool, optional (default=False)
We use ``tqdm`` for logging, which will print a nice progress bar that updates in place
after every batch. This is nice if you're running training on a local shell, but can
cause problems with log files from, e.g., a docker image running on kubernetes. If
``no_tqdm`` is ``True``, we will not use tqdm, and instead log batch statistics using
``logger.info``.
"""
self._model = model
parameters_to_train = [(n, p)
for n, p in self._model.named_parameters()
if p.requires_grad]
self._task_list = task_list
self._n_tasks = len(self._task_list)
self._optimizer_params = optimizer_params
self._optimizers = {}
self._lr_scheduler_params = lr_scheduler_params
self._schedulers = {}
for task in self._task_list:
task_name = task._name
self._optimizers[task_name] = Optimizer.from_params(
model_parameters=parameters_to_train,
params=deepcopy(optimizer_params))
self._schedulers[task_name] = LearningRateScheduler.from_params(
optimizer=self._optimizers[task_name],
params=deepcopy(lr_scheduler_params))
self._serialization_dir = serialization_dir
self._patience = patience
self._num_epochs = num_epochs
self._cuda_device = cuda_device
if self._cuda_device >= 0:
check_for_gpu(self._cuda_device)
self._model = self._model.cuda(self._cuda_device)
self._gradient_accumulation_steps = gradient_accumulation_steps
self._grad_norm = grad_norm
self._grad_clipping = grad_clipping
self._min_lr = min_lr
self._task_infos = None
self._metric_infos = None
self._tr_generators = None
self._no_tqdm = no_tqdm
self._summary_interval = summary_interval # num batches between logging to tensorboard
# self._log_parameter_statistics = log_parameter_statistics
# self._log_gradient_statistics = log_gradient_statistics
self._global_step = 0
# train_log = SummaryWriter(os.path.join(self._serialization_dir, "log", "train"))
# validation_log = SummaryWriter(os.path.join(self._serialization_dir, "log", "validation"))
# self._tensorboard = TensorboardWriter(train_log=train_log, validation_log=validation_log)
self._batch_num_total = 0
self._tensorboard = TensorboardWriter(
get_batch_num_total=lambda: self._batch_num_total,
serialization_dir=serialization_dir,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_learning_rate=True)
# Enable activation logging.
if histogram_interval is not None:
self._tensorboard.enable_activation_logging(self._model)
def _get_optimizer(self, lr: float = 1.0):
return Optimizer.from_params(self.model.named_parameters(), Params({"type": "sgd", "lr": lr}))
def from_params(
cls, # type: ignore
params: Params,
serialization_dir: str,
recover: bool = False) -> 'Trainer':
# modified for second training_data
all_datasets = datasets_from_params(params)
# copied from allennlp.training.trainer.TrainingPieces
# modified for second training_data
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
if recover and os.path.exists(
os.path.join(serialization_dir, "vocabulary")):
vocab = Vocabulary.from_files(
os.path.join(serialization_dir, "vocabulary"))
params.pop("vocabulary", {})
else:
vocab = Vocabulary.from_params(params.pop(
"vocabulary", {}), (instance
for key, dataset in all_datasets.items()
for instance in dataset
if key in datasets_for_vocab_creation))
model = Model.from_params(vocab=vocab, params=params.pop('model'))
model.extend_embedder_vocab()
vocab.save_to_files(os.path.join(serialization_dir, "vocabulary"))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(model.vocab)
validation_iterator_params = params.pop("validation_iterator", None)
if validation_iterator_params:
validation_iterator = DataIterator.from_params(
validation_iterator_params)
validation_iterator.index_with(model.vocab)
else:
validation_iterator = None
train_data = all_datasets['train']
validation_data = all_datasets.get('validation')
test_data = all_datasets.get('test')
train_low_data = all_datasets.get('train_low')
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
frozen_parameter_names, tunable_parameter_names = \
get_frozen_and_tunable_parameter_names(model)
logger.info("Following parameters are Frozen (without gradient):")
for name in frozen_parameter_names:
logger.info(name)
logger.info("Following parameters are Tunable (with gradient):")
for name in tunable_parameter_names:
logger.info(name)
# END OF TrainerPieces code
params = trainer_params
# pylint: disable=arguments-differ
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
momentum_scheduler_params = params.pop("momentum_scheduler", None)
if isinstance(cuda_device, list):
model_device = cuda_device[0]
else:
model_device = cuda_device
if model_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(model_device)
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if "moving_average" in params:
moving_average = MovingAverage.from_params(
params.pop("moving_average"), parameters=parameters)
else:
moving_average = None
if lr_scheduler_params:
lr_scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params)
else:
lr_scheduler = None
if momentum_scheduler_params:
momentum_scheduler = MomentumScheduler.from_params(
optimizer, momentum_scheduler_params)
else:
momentum_scheduler = None
if 'checkpointer' in params:
if 'keep_serialized_model_every_num_seconds' in params or \
'num_serialized_models_to_keep' in params:
raise ConfigurationError(
"Checkpointer may be initialized either from the 'checkpointer' key or from the "
"keys 'num_serialized_models_to_keep' and 'keep_serialized_model_every_num_seconds'"
" but the passed config uses both methods.")
checkpointer = Checkpointer.from_params(params.pop("checkpointer"))
else:
num_serialized_models_to_keep = params.pop_int(
"num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
checkpointer = Checkpointer(
serialization_dir=serialization_dir,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=
keep_serialized_model_every_num_seconds)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool(
"should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate",
False)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
epoch_low_start = params.pop_int("epoch_low_start", None)
epoch_without_improvement_low_start = params.pop_int(
"epoch_without_improvement_low_start", None)
params.assert_empty(cls.__name__)
return cls(
model,
optimizer,
iterator,
train_data,
validation_data,
train_low_dataset=train_low_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=lr_scheduler,
momentum_scheduler=momentum_scheduler,
checkpointer=checkpointer,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period,
moving_average=moving_average,
epoch_low_start=epoch_low_start,
epoch_without_improvement_low_start=
epoch_without_improvement_low_start,
)
def test_checkpointing_does_run(self, build_trainer_params_function):
# Check that checkpointing does run and does sanity checks that at each step
# it saves the most recent checkpoint as well as the best checkpoint
# correctly for both pretrain and target_train stages.
with mock.patch("jiant.models.MultiTaskModel") as MockModel:
import torch
import copy
import time
from allennlp.common.params import Params
MockModel.return_value.eval.return_value = None
MockModel.return_value.state_dict.return_value = {
"model1": {
"requires_grad": True
}
}
pad_dict = self.wic.val_data[0].get_padding_lengths()
sorting_keys = []
for field in pad_dict:
for pad_field in pad_dict[field]:
sorting_keys.append((field, pad_field))
iterator = BucketIterator(
sorting_keys=sorting_keys,
max_instances_in_memory=10000,
batch_size=4,
biggest_batch_first=True,
)
opt_params = Params({"type": "adam", "lr": 1e-05})
opt_params2 = copy.deepcopy(opt_params)
scheduler_params = Params({
"type": "reduce_on_plateau",
"factor": 0.05,
"mode": "max",
"patience": 4,
"threshold": 0.05,
"threshold_mode": "abs",
"verbose": True,
})
train_params = [
(
"_text_field_embedder.model.encoder.layer.9.output.dense.bias",
torch.Tensor([0.1, 0.3, 0.4, 0.8]),
),
("sent_encoder.layer.1", torch.Tensor([0.1, 0.3, 0.4, 0.8])),
("type", torch.Tensor([0.1])),
]
scheduler = LearningRateScheduler.from_params(
Optimizer.from_params(train_params, opt_params2),
copy.deepcopy(scheduler_params))
optimizer = Optimizer.from_params(train_params,
copy.deepcopy(opt_params))
_task_infos = {
"wic": {
"iterator": iterator(self.wic.val_data, num_epochs=1),
"n_tr_batches": 1,
"loss": 0.0,
"tr_generator": iterator(self.wic.val_data, num_epochs=1),
"total_batches_trained": 400,
"n_batches_since_val": 0,
"optimizer": optimizer,
"scheduler": scheduler,
"stopped": False,
"last_log": time.time(),
}
}
_metric_infos = {
metric: {
"hist": [],
"stopped": False,
"best": (-1, {})
}
for metric in [self.wic.val_metric]
}
MockModel.return_value._setup_training.return_value = _task_infos, _metric_infos
class MockParams:
def __init__(self, requires_grad):
self.requires_grad = requires_grad
MockModel.return_value.named_parameters.return_value = [
("model1", MockParams(True))
]
MockModel.use_bert = 1
model = MockModel()
pt_trainer, _, _, _ = trainer.build_trainer(
self.args,
[
"wic"
], # here, we use WIC twice to reduce the amount of boiler-plate code
model,
self.args.run_dir,
self.wic.val_metric_decreases,
phase="pretrain",
)
tt_trainer, _, _, _ = trainer.build_trainer(
self.args,
["wic"],
model,
self.args.run_dir,
self.wic.val_metric_decreases,
phase="target_train",
)
os.mkdir(os.path.join(self.temp_dir, "wic"))
tt_trainer.task_to_metric_mapping = {
self.wic.val_metric: self.wic.name
}
pt_trainer._task_infos = _task_infos
pt_trainer._metric_infos = _metric_infos
pt_trainer._optimizer = optimizer
pt_trainer._scheduler = scheduler
pt_trainer._save_checkpoint(
{
"step": 10,
"validation_pass": 1,
"should_stop": 0
},
tasks=[self.wic],
phase="pretrain",
new_best=True,
)
pt_trainer._save_checkpoint(
{
"step": 10,
"validation_pass": 2,
"should_stop": 0
},
tasks=[self.wic],
phase="pretrain",
new_best=True,
)
tt_trainer._task_infos = _task_infos
tt_trainer._metric_infos = _metric_infos
tt_trainer._optimizer = optimizer
tt_trainer._scheduler = scheduler
tt_trainer._save_checkpoint(
{
"step": 10,
"validation_pass": 1,
"should_stop": 0
},
tasks=[self.wic],
phase="target_train",
new_best=True,
)
tt_trainer._save_checkpoint(
{
"step": 10,
"validation_pass": 2,
"should_stop": 0
},
tasks=[self.wic],
phase="target_train",
new_best=False,
)
assert (os.path.exists(
os.path.join(self.temp_dir, "wic",
"model_state_target_train_val_1.best.th"))
and os.path.exists(
os.path.join(self.temp_dir, "wic",
"model_state_target_train_val_2.th"))
and os.path.exists(
os.path.join(self.temp_dir,
"model_state_pretrain_val_2.best.th"))
and os.path.exists(
os.path.join(self.temp_dir,
"model_state_pretrain_val_1.th")))
# Assert only one checkpoint is created for pretrain stage.
pretrain_best_checkpoints = glob.glob(
os.path.join(self.temp_dir,
"model_state_pretrain_val_*.best.th"))
assert len(pretrain_best_checkpoints) == 1
def test_no_metric_wrapper_can_support_none_for_metrics(self):
lrs = LearningRateScheduler.from_params(Optimizer.from_params(self.model.named_parameters(),
Params({"type": "adam"})),
Params({"type": "step", "step_size": 1}))
lrs.step(None, None)
def test_noam_learning_rate_schedule_does_not_crash(self):
lrs = LearningRateScheduler.from_params(Optimizer.from_params(self.model.named_parameters(),
Params({"type": "adam"})),
Params({"type": "noam", "model_size": 10, "warmup_steps": 2000}))
lrs.step(None, None)
lrs.step_batch(None)
def test_reduce_on_plateau_error_throw_when_mode_not_specified(self):
with self.assertRaises(ConfigurationError) as context:
LearningRateScheduler.from_params(Optimizer.from_params(self.model.named_parameters(),
Params({"type": "adam"})),
Params({"type": "reduce_on_plateau"})).step(None, None)
assert "ReduceLROnPlateau requires a mode to be specified" in str(context.exception)
def from_params(
cls, # type: ignore
params: Params,
serialization_dir: str,
recover: bool = False,
cache_directory: str = None,
cache_prefix: str = None) -> 'Trainer':
# datasets = meta_dataset_from_params(params, cache_directory=cache_directory, cache_prefix=cache_prefix)
# model = Model.from_params(vocab=vocab, params=params.pop("model"))
# iterator = DataIterator.from_params(params.pop("iterator"))
# iterator.index_with(model.vocab)
pieces = MetaTrainerPieces.from_params(params, serialization_dir,
recover, cache_directory,
cache_prefix)
model = pieces.model
iterator = pieces.iterator,
# params=pieces.params,
train_data = pieces.train_dataset
validation_data = pieces.validation_dataset
validation_iterator = pieces.validation_iterator
params = pieces.params
# pylint: disable=arguments-differ
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", [0, 1]))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
momentum_scheduler_params = params.pop("momentum_scheduler", None)
if isinstance(cuda_device, list):
model_device = cuda_device[0]
else:
model_device = cuda_device
if model_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(model_device)
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if "moving_average" in params:
moving_average = MovingAverage.from_params(
params.pop("moving_average"), parameters=parameters)
else:
moving_average = None
if lr_scheduler_params:
lr_scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params)
else:
lr_scheduler = None
if momentum_scheduler_params:
momentum_scheduler = MomentumScheduler.from_params(
optimizer, momentum_scheduler_params)
else:
momentum_scheduler = None
if 'checkpointer' in params:
if 'keep_serialized_model_every_num_seconds' in params or \
'num_serialized_models_to_keep' in params:
raise ConfigurationError(
"Checkpointer may be initialized either from the 'checkpointer' key or from the "
"keys 'num_serialized_models_to_keep' and 'keep_serialized_model_every_num_seconds'"
" but the passed config uses both methods.")
checkpointer = Checkpointer.from_params(params.pop("checkpointer"))
else:
num_serialized_models_to_keep = params.pop_int(
"num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
checkpointer = Checkpointer(
serialization_dir=serialization_dir,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=
keep_serialized_model_every_num_seconds)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool(
"should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate",
False)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
print('[info] cuda_device in metatrainer.from_param is:{}'.format(
cuda_device))
params.assert_empty(cls.__name__)
return cls(
model,
optimizer,
iterator,
train_data,
validation_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=lr_scheduler,
momentum_scheduler=momentum_scheduler,
checkpointer=checkpointer,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period,
moving_average=moving_average,
# distributed=distributed,
# rank=local_rank,
# world_size=world_size,
# num_gradient_accumulation_steps=num_gradient_accumulation_steps,
)
def _get_optimizer(self):
return Optimizer.from_params(
model_parameters=self.model.named_parameters(),
params=Params({"type": "sgd", "lr": 1.0, "momentum": self.base_momentum}),
)
model_fp = str(Path('.', 'model_configs', 'standard', 'ds_elmo_t_fine_tune_laptop.json').resolve())
params = Params.from_file(model_fp)
data_fp = str(Path('data', 'splits', 'Laptop Test').resolve())
reader = DatasetReader.from_params(params['dataset_reader'])
instances = list(reader.read(data_fp))
if 'vocabulary' in params:
vocab_params = params['vocabulary']
vocab = Vocabulary.from_params(params=vocab_params, instances=instances)
else:
vocab = Vocabulary.from_instances(instances)
import re
model = Model.from_params(vocab=vocab, params=params['model'])
names = [name for name, param in model.named_parameters()]
optim = Optimizer.from_params(model.named_parameters(),params['trainer']['optimizer'])
values = {}
for i, param_group in enumerate(reversed(optim.param_groups)):
values[i] = param_group['params']
encoder = set()
project = set()
embedder_5 = set()
embedder_4 = set()
embedder_3 = set()
embedder_2 = set()
embedder_1 = set()
embedder_0 = set()
embedder_char = set()
embedding_scalar_mix = set()
other = set()
def _get_optimizer(self, lr: float = 1.0):
optimizer_params = Params({"type": "sgd", "lr": lr})
optimizer_params["parameter_groups"] = [[[f"^{m}"], {}]
for m in self.model._modules]
return Optimizer.from_params(self.model.named_parameters(),
optimizer_params)
train_loader = VCRLoader.from_dataset(train, **loader_params)
val_loader = VCRLoader.from_dataset(val, **loader_params)
test_loader = VCRLoader.from_dataset(test, **loader_params)
ARGS_RESET_EVERY = 100
print("Loading {} for {}".format(params['model'].get('type', 'WTF?'),
'rationales' if args.rationale else 'answer'),
flush=True)
model = Model.from_params(vocab=train.vocab, params=params['model'])
# for submodule in model.detector.backbone.modules():
# if isinstance(submodule, BatchNorm2d):
# submodule.track_running_stats = False
model = DataParallel(model).cuda() if NUM_GPUS > 1 else model.cuda()
optimizer = Optimizer.from_params(
[x for x in model.named_parameters() if x[1].requires_grad],
params['trainer']['optimizer'])
lr_scheduler_params = params['trainer'].pop("learning_rate_scheduler", None)
scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params) if lr_scheduler_params else None
if os.path.exists(args.folder) and args.restore:
print('restore is True')
print("Found folder! restoring", flush=True)
start_epoch, val_metric_per_epoch = restore_checkpoint(
model,
optimizer,
serialization_dir=args.folder,
learning_rate_scheduler=scheduler)
else:
def from_params(
cls, # type: ignore
params: Params,
serialization_dir: str,
recover: bool = False,
cache_directory: str = None,
cache_prefix: str = None) -> 'SloppyTrainer':
# pylint: disable=arguments-differ
from allennlp.training.trainer_pieces import TrainerPieces
pieces = TrainerPieces.from_params(params, serialization_dir, recover) # pylint: disable=no-member
model = pieces.model
serialization_dir = serialization_dir
iterator = pieces.iterator
train_data = pieces.train_dataset
validation_data = pieces.validation_dataset
params = pieces.params
validation_iterator = pieces.validation_iterator
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
momentum_scheduler_params = params.pop("momentum_scheduler", None)
if isinstance(cuda_device, list):
model_device = cuda_device[0]
else:
model_device = cuda_device
if model_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(model_device)
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if "moving_average" in params:
moving_average = MovingAverage.from_params(
params.pop("moving_average"), parameters=parameters)
else:
moving_average = None
if lr_scheduler_params:
lr_scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params)
else:
lr_scheduler = None
if momentum_scheduler_params:
momentum_scheduler = MomentumScheduler.from_params(
optimizer, momentum_scheduler_params)
else:
momentum_scheduler = None
if 'checkpointer' in params:
if 'keep_serialized_model_every_num_seconds' in params or \
'num_serialized_models_to_keep' in params:
raise ConfigurationError(
"Checkpointer may be initialized either from the 'checkpointer' key or from the "
"keys 'num_serialized_models_to_keep' and 'keep_serialized_model_every_num_seconds'"
" but the passed config uses both methods.")
checkpointer = Checkpointer.from_params(params.pop("checkpointer"))
else:
num_serialized_models_to_keep = params.pop_int(
"num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
checkpointer = Checkpointer(
serialization_dir=serialization_dir,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=
keep_serialized_model_every_num_seconds)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool(
"should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate",
False)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
mixed_precision = params.pop_bool("mixed_precision", False)
params.assert_empty(cls.__name__)
return cls(
model,
optimizer,
iterator,
train_data,
validation_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=lr_scheduler,
momentum_scheduler=momentum_scheduler,
checkpointer=checkpointer,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period,
moving_average=moving_average,
mixed_precision=mixed_precision)
def from_params(
cls, # type: ignore
model: Model,
serialization_dir: str,
files_to_archive: Dict[str, str],
iterator: DataIterator,
train_data: Iterable[Instance],
validation_data: Optional[Iterable[Instance]],
params: Params,
validation_iterator: DataIterator = None) -> 'TrainerFP16':
# pylint: disable=arguments-differ
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
momentum_scheduler_params = params.pop("momentum_scheduler", None)
fp16 = params.pop_bool("fp16", False)
dynamic_loss_scale = params.pop_bool("dynamic_loss_scale", True)
validate_first = params.pop_bool("validate_first", False)
if isinstance(cuda_device, list):
model_device = cuda_device[0]
else:
model_device = cuda_device
if fp16:
model.half()
if model_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(model_device)
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
# If fp16, need to wrap the optimizer
try:
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if fp16:
# The FP16_Optimizer we use depends on whether the optimizer is FusedAdam or a regular pytorch optimizer
if isinstance(optimizer, FusedAdam):
from apex.optimizers import FP16_Optimizer
else:
from apex.fp16_utils import FP16_Optimizer
optimizer = FP16_Optimizer(optimizer,
dynamic_loss_scale=dynamic_loss_scale)
if "moving_average" in params:
moving_average = MovingAverage.from_params(
params.pop("moving_average"), parameters=parameters)
else:
moving_average = None
if lr_scheduler_params:
lr_scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params)
else:
lr_scheduler = None
if momentum_scheduler_params:
momentum_scheduler = MomentumScheduler.from_params(
optimizer, momentum_scheduler_params)
else:
momentum_scheduler = None
if 'checkpointer' in params:
if 'keep_serialized_model_every_num_seconds' in params or \
'num_serialized_models_to_keep' in params:
raise ConfigurationError(
"Checkpointer may be initialized either from the 'checkpointer' key or from the "
"keys 'num_serialized_models_to_keep' and 'keep_serialized_model_every_num_seconds'"
" but the passed config uses both methods.")
checkpointer = Checkpointer.from_params(params.pop("checkpointer"))
else:
num_serialized_models_to_keep = params.pop_int(
"num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
checkpointer = Checkpointer(
serialization_dir=serialization_dir,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=
keep_serialized_model_every_num_seconds)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool(
"should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate",
False)
statistics_interval = params.pop_int("statistics_interval", 5000)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
params.assert_empty(cls.__name__)
return cls(
model,
optimizer,
iterator,
train_data,
validation_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=lr_scheduler,
momentum_scheduler=momentum_scheduler,
checkpointer=checkpointer,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
statistics_interval=statistics_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period,
moving_average=moving_average,
fp16=fp16,
validate_first=validate_first,
files_to_archive=files_to_archive)
def from_params( # type: ignore
cls,
params: Params,
serialization_dir: str,
recover: bool = False,
local_rank: int = 0,
) -> "MetaTrainer":
from allennlp.training.trainer import Trainer
from src.training.trainer_pieces import MetaTrainerPieces
config = dict(as_flat_dict(params.as_dict()))
pieces = MetaTrainerPieces.from_params(params, serialization_dir, recover)
model = pieces.model
serialization_dir = serialization_dir
iterator = pieces.iterator
train_datas = pieces.train_datasets
validation_datas = pieces.validation_datasets
params = pieces.params
validation_iterator = pieces.validation_iterator
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
momentum_scheduler_params = params.pop("momentum_scheduler", None)
check_for_gpu(cuda_device)
if cuda_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(cuda_device)
parameters = [[n, p] for n, p in model.named_parameters() if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if "moving_average" in params:
moving_average = MovingAverage.from_params(
params.pop("moving_average"), parameters=parameters
)
else:
moving_average = None
if lr_scheduler_params:
lr_scheduler = LearningRateScheduler.from_params(optimizer, lr_scheduler_params)
else:
lr_scheduler = None
if momentum_scheduler_params:
momentum_scheduler = MomentumScheduler.from_params(optimizer, momentum_scheduler_params)
else:
momentum_scheduler = None
if "checkpointer" in params:
if (
"keep_serialized_model_every_num_seconds" in params
or "num_serialized_models_to_keep" in params
):
raise ConfigurationError(
"Checkpointer may be initialized either from the 'checkpointer' key or from the "
"keys 'num_serialized_models_to_keep' and 'keep_serialized_model_every_num_seconds'"
" but the passed config uses both methods."
)
checkpointer = Checkpointer.from_params(params.pop("checkpointer"))
else:
num_serialized_models_to_keep = params.pop_int("num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None
)
checkpointer = Checkpointer(
serialization_dir=serialization_dir,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=keep_serialized_model_every_num_seconds,
)
log_grad_norm = params.pop("log_grad_norm", "total")
save_embedder = params.pop_bool("save_embedder", True)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool("should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate", False)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
distributed = params.pop_bool("distributed", False)
world_size = params.pop_int("world_size", 1)
num_gradient_accumulation_steps = params.pop("num_gradient_accumulation_steps", 1)
tasks_per_step = params.pop_int("tasks_per_step", 0)
wrapper = Wrapper.from_params(
params.pop("wrapper"),
model=model,
meta_optimizer=optimizer,
)
task_discriminator_params = params.pop("task_discriminator", None)
if task_discriminator_params:
num_tasks = model.vocab.get_vocab_size("lang_labels")
task_discriminator = TaskDiscriminator.from_params(task_discriminator_params,
num_tasks=num_tasks)
if cuda_device >= 0:
task_discriminator = task_discriminator.cuda(cuda_device)
discriminator_parameters = \
[[n, p] for n, p in task_discriminator.named_parameters() if p.requires_grad]
discriminator_optimizer = Optimizer.from_params(discriminator_parameters,
params.pop("discriminator_optimizer"))
else:
task_discriminator = None
discriminator_optimizer = None
writer = None
wandb_config = params.pop("wandb", None)
if wandb_config is not None:
writer = WandBWriter(config, wrapper.container, wandb_config)
params.assert_empty(cls.__name__)
return cls(
model,
optimizer,
iterator,
train_datas,
validation_datas,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
save_embedder=save_embedder,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=lr_scheduler,
momentum_scheduler=momentum_scheduler,
checkpointer=checkpointer,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period,
moving_average=moving_average,
distributed=distributed,
local_rank=local_rank,
world_size=world_size,
num_gradient_accumulation_steps=num_gradient_accumulation_steps,
log_grad_norm=log_grad_norm,
wrapper=wrapper,
task_discriminator=task_discriminator,
discriminator_optimizer=discriminator_optimizer,
tasks_per_step=tasks_per_step,
writer=writer,
)
def __init__(
self,
model: Model,
optimizer,
iterator: DataIterator,
train_dataset: Iterable[Instance],
validation_dataset: Optional[Iterable[Instance]] = None,
patience: Optional[int] = None,
validation_metric: str = "-loss",
validation_iterator: DataIterator = None,
shuffle: bool = True,
num_epochs: int = 20,
serialization_dir: Optional[str] = None,
num_serialized_models_to_keep: int = 20,
keep_serialized_model_every_num_seconds: int = None,
checkpointer: Checkpointer = None,
model_save_interval: float = None,
cuda_device: Union[int, List] = -1,
grad_norm: Optional[float] = None,
grad_clipping: Optional[float] = None,
learning_rate_scheduler=None,
momentum_scheduler=None,
summary_interval: int = 100,
histogram_interval: int = None,
should_log_parameter_statistics: bool = True,
should_log_learning_rate: bool = False,
log_batch_size_period: Optional[int] = None,
moving_average=None,
) -> None:
"""
A trainer for doing supervised learning. It just takes a labeled dataset
and a ``DataIterator``, and uses the supplied ``Optimizer`` to learn the weights
for your model over some fixed number of epochs. You can also pass in a validation
dataset and enable early stopping. There are many other bells and whistles as well.
Parameters
----------
model : ``Model``, required.
An AllenNLP model to be optimized. Pytorch Modules can also be optimized if
their ``forward`` method returns a dictionary with a "loss" key, containing a
scalar tensor representing the loss function to be optimized.
If you are training your model using GPUs, your model should already be
on the correct device. (If you use `Trainer.from_params` this will be
handled for you.)
optimizer : ``torch.nn.Optimizer``, required.
An instance of a Pytorch Optimizer, instantiated with the parameters of the
model to be optimized.
iterator : ``DataIterator``, required.
A method for iterating over a ``Dataset``, yielding padded indexed batches.
train_dataset : ``Dataset``, required.
A ``Dataset`` to train on. The dataset should have already been indexed.
validation_dataset : ``Dataset``, optional, (default = None).
A ``Dataset`` to evaluate on. The dataset should have already been indexed.
patience : Optional[int] > 0, optional (default=None)
Number of epochs to be patient before early stopping: the training is stopped
after ``patience`` epochs with no improvement. If given, it must be ``> 0``.
If None, early stopping is disabled.
validation_metric : str, optional (default="loss")
Validation metric to measure for whether to stop training using patience
and whether to serialize an ``is_best`` model each epoch. The metric name
must be prepended with either "+" or "-", which specifies whether the metric
is an increasing or decreasing function.
validation_iterator : ``DataIterator``, optional (default=None)
An iterator to use for the validation set. If ``None``, then
use the training `iterator`.
shuffle: ``bool``, optional (default=True)
Whether to shuffle the instances in the iterator or not.
num_epochs : int, optional (default = 20)
Number of training epochs.
serialization_dir : str, optional (default=None)
Path to directory for saving and loading model files. Models will not be saved if
this parameter is not passed.
num_serialized_models_to_keep : ``int``, optional (default=20)
Number of previous model checkpoints to retain. Default is to keep 20 checkpoints.
A value of None or -1 means all checkpoints will be kept.
keep_serialized_model_every_num_seconds : ``int``, optional (default=None)
If num_serialized_models_to_keep is not None, then occasionally it's useful to
save models at a given interval in addition to the last num_serialized_models_to_keep.
To do so, specify keep_serialized_model_every_num_seconds as the number of seconds
between permanently saved checkpoints. Note that this option is only used if
num_serialized_models_to_keep is not None, otherwise all checkpoints are kept.
checkpointer : ``Checkpointer``, optional (default=None)
An instance of class Checkpointer to use instead of the default. If a checkpointer is specified,
the arguments num_serialized_models_to_keep and keep_serialized_model_every_num_seconds should
not be specified. The caller is responsible for initializing the checkpointer so that it is
consistent with serialization_dir.
model_save_interval : ``float``, optional (default=None)
If provided, then serialize models every ``model_save_interval``
seconds within single epochs. In all cases, models are also saved
at the end of every epoch if ``serialization_dir`` is provided.
cuda_device : ``Union[int, List[int]]``, optional (default = -1)
An integer or list of integers specifying the CUDA device(s) to use. If -1, the CPU is used.
grad_norm : ``float``, optional, (default = None).
If provided, gradient norms will be rescaled to have a maximum of this value.
grad_clipping : ``float``, optional (default = ``None``).
If provided, gradients will be clipped `during the backward pass` to have an (absolute)
maximum of this value. If you are getting ``NaNs`` in your gradients during training
that are not solved by using ``grad_norm``, you may need this.
learning_rate_scheduler : ``LearningRateScheduler``, optional (default = None)
If specified, the learning rate will be decayed with respect to
this schedule at the end of each epoch (or batch, if the scheduler implements
the ``step_batch`` method). If you use :class:`torch.optim.lr_scheduler.ReduceLROnPlateau`,
this will use the ``validation_metric`` provided to determine if learning has plateaued.
To support updating the learning rate on every batch, this can optionally implement
``step_batch(batch_num_total)`` which updates the learning rate given the batch number.
momentum_scheduler : ``MomentumScheduler``, optional (default = None)
If specified, the momentum will be updated at the end of each batch or epoch
according to the schedule.
summary_interval: ``int``, optional, (default = 100)
Number of batches between logging scalars to tensorboard
histogram_interval : ``int``, optional, (default = ``None``)
If not None, then log histograms to tensorboard every ``histogram_interval`` batches.
When this parameter is specified, the following additional logging is enabled:
* Histograms of model parameters
* The ratio of parameter update norm to parameter norm
* Histogram of layer activations
We log histograms of the parameters returned by
``model.get_parameters_for_histogram_tensorboard_logging``.
The layer activations are logged for any modules in the ``Model`` that have
the attribute ``should_log_activations`` set to ``True``. Logging
histograms requires a number of GPU-CPU copies during training and is typically
slow, so we recommend logging histograms relatively infrequently.
Note: only Modules that return tensors, tuples of tensors or dicts
with tensors as values currently support activation logging.
should_log_parameter_statistics : ``bool``, optional, (default = True)
Whether to send parameter statistics (mean and standard deviation
of parameters and gradients) to tensorboard.
should_log_learning_rate : ``bool``, optional, (default = False)
Whether to send parameter specific learning rate to tensorboard.
log_batch_size_period : ``int``, optional, (default = ``None``)
If defined, how often to log the average batch size.
moving_average: ``MovingAverage``, optional, (default = None)
If provided, we will maintain moving averages for all parameters. During training, we
employ a shadow variable for each parameter, which maintains the moving average. During
evaluation, we backup the original parameters and assign the moving averages to corresponding
parameters. Be careful that when saving the checkpoint, we will save the moving averages of
parameters. This is necessary because we want the saved model to perform as well as the validated
model if we load it later. But this may cause problems if you restart the training from checkpoint.
"""
super().__init__(serialization_dir, cuda_device)
# I am not calling move_to_gpu here, because if the model is
# not already on the GPU then the optimizer is going to be wrong.
self.model = model
self.iterator = iterator
self._validation_iterator = validation_iterator
self.shuffle = shuffle
optimiser_params = optimizer
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
from copy import deepcopy
self.optimizer = Optimizer.from_params(parameters,
deepcopy(optimiser_params))
self.optimizer_lang1 = Optimizer.from_params(
parameters, deepcopy(optimiser_params))
self.optimizer_lang2 = Optimizer.from_params(
parameters, deepcopy(optimiser_params))
self.optimizer_cm = Optimizer.from_params(parameters,
deepcopy(optimiser_params))
self.train_data = train_dataset
self._validation_data = validation_dataset
if patience is None: # no early stopping
if validation_dataset:
logger.warning(
"You provided a validation dataset but patience was set to None, "
"meaning that early stopping is disabled")
elif (not isinstance(patience, int)) or patience <= 0:
raise ConfigurationError(
'{} is an invalid value for "patience": it must be a positive integer '
"or None (if you want to disable early stopping)".format(
patience))
# For tracking is_best_so_far and should_stop_early
self._metric_tracker = MetricTracker(patience, validation_metric)
# Get rid of + or -
self._validation_metric = validation_metric[1:]
self._num_epochs = num_epochs
if checkpointer is not None:
# We can't easily check if these parameters were passed in, so check against their default values.
# We don't check against serialization_dir since it is also used by the parent class.
if (num_serialized_models_to_keep != 20
or keep_serialized_model_every_num_seconds is not None):
raise ConfigurationError(
"When passing a custom Checkpointer, you may not also pass in separate checkpointer "
"args 'num_serialized_models_to_keep' or 'keep_serialized_model_every_num_seconds'."
)
self._checkpointer = checkpointer
else:
self._checkpointer = Checkpointer(
serialization_dir,
keep_serialized_model_every_num_seconds,
num_serialized_models_to_keep,
)
self._model_save_interval = model_save_interval
self._grad_norm = grad_norm
self._grad_clipping = grad_clipping
if learning_rate_scheduler:
self._learning_rate_scheduler = LearningRateScheduler.from_params(
self.optimizer, deepcopy(learning_rate_scheduler))
self._learning_rate_scheduler_lang1 = LearningRateScheduler.from_params(
self.optimizer_lang1, deepcopy(learning_rate_scheduler))
self._learning_rate_scheduler_lang2 = LearningRateScheduler.from_params(
self.optimizer_lang2, deepcopy(learning_rate_scheduler))
self._learning_rate_scheduler_cm = LearningRateScheduler.from_params(
self.optimizer_cm, deepcopy(learning_rate_scheduler))
else:
self._learning_rate_scheduler, self._learning_rate_scheduler_lang1, self._learning_rate_scheduler_lang2, self._learning_rate_scheduler_cm = None, None, None, None
if momentum_scheduler:
self._momentum_scheduler = MomentumScheduler.from_params(
self.optimizer, deepcopy(momentum_scheduler))
self._momentum_scheduler_lang1 = MomentumScheduler.from_params(
self.optimizer_lang1, deepcopy(momentum_scheduler))
self._momentum_scheduler_lang2 = MomentumScheduler.from_params(
self.optimizer_lang2, deepcopy(momentum_scheduler))
self._momentum_scheduler_cm = MomentumScheduler.from_params(
self.optimizer_lang3, deepcopy(momentum_scheduler))
else:
self._momentum_scheduler, self._momentum_scheduler_lang1, self._momentum_scheduler_lang2, self._momentum_scheduler_cm = None, None, None, None
# WE HAVE NOT USED IT YET
self._moving_average, self.moving_average_lang1, self.moving_average_lang2, self.moving_average_cm = None, None, None, None
# We keep the total batch number as an instance variable because it
# is used inside a closure for the hook which logs activations in
# ``_enable_activation_logging``.
self._batch_num_total = 0
self._tensorboard = TensorboardWriter(
get_batch_num_total=lambda: self._batch_num_total,
serialization_dir=serialization_dir,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
)
self._log_batch_size_period = log_batch_size_period
self._last_log = 0.0 # time of last logging
# Enable activation logging.
if histogram_interval is not None:
self._tensorboard.enable_activation_logging(self.model)
def main(args):
params = Params.from_file(args.params_file)
# print('Data seed:{}, Percent data: {}'.format(shuffle_id, train_size))
settings.cuda = params['cuda_device'] != -1
common_util.prepare_environment(params)
serialization_dir = params['serialization_dir']
training_util.create_serialization_dir(params, serialization_dir,
args.recover, args.force)
common_util.prepare_global_logging(serialization_dir, True)
logging.info(
"torch version: {}, allennlp version: {}, allennlp path: {}".format(
torch.__version__, allennlp.__version__, allennlp.__path__))
params.to_file(os.path.join(serialization_dir, CONFIG_NAME))
semi_supervision = params.get('semi_supervised', False)
which_mixer = params.get('which_mixer', 'cm')
dd_warmup_iters = params.pop('dd_warmup_iters', 1)
dd_semi_warmup_iters = params.pop('dd_semi_warmup_iters', 1)
dd_update_freq = params.pop('dd_update_freq', 2)
constraints_wt = params.get('constraints_wt', 0)
calc_valid_freq = params.get('calc_valid_freq', 1)
backprop_after_xbatches = params.pop('backprop_after_xbatches', 1)
min_pct_of_unlabelled = params.pop('min_pct_of_unlabelled', 0.0)
dd_increase_freq_after = params.pop('dd_increase_freq_after', 0)
dd_increase_freq_by = params.pop('dd_increase_freq_by', 0)
dd_decay_lr = params.pop('dd_decay_lr', 0)
dd_decay_lr_after = params.pop('dd_decay_lr_after', 1.0)
grad_norm_before_warmup = params.pop('grad_norm_before_warmup', 0)
if semi_supervision:
print("Semi Supervision On")
for key in [
'warmup_epochs', 'unlabelled_train_data_file', 'test_data_file',
'data_dir', 'cuda_device', 'serialization_dir', 'train_data_file',
'validation_data_file', 'constraints_wt', 'train_size',
'shuffle_id', 'semi_supervised', 'which_mixer',
'distributed_lambda_update', 'calc_valid_freq'
]:
params.pop(key, None)
print("Trainer pieces")
pieces = gan_trainer.TrainerPiecesForSemi.from_params(
params, serialization_dir, args.recover, semi_supervision) # pylint: disable=no-member
#pieces for constrained learning"
print("Constraint model")
constraints_model = Model.from_params(vocab=pieces.model.vocab,
params=params.pop('dd_constraints'))
dd_params = [[n, p] for n, p in constraints_model.named_parameters()
if p.requires_grad]
dd_optimizer = None
dd_optim_params = params.pop('dd_optimizer', None)
if len(dd_params) > 0:
dd_optimizer = Optimizer.from_params(dd_params, dd_optim_params)
cp = None
chfile = None
#Pdb().set_trace()
if args.weight_dir is not None:
#Pdb().set_trace()
flag = True
if args.weight_file is not None:
logging.info("Loading Model weights from :{}".format(
os.path.join(args.weight_dir, args.weight_file)))
model_states = torch.load(
os.path.join(args.weight_dir, args.weight_file))
pieces.model.load_state_dict(model_states)
flag = False
if args.dd_file is not None:
logging.info("Loading Constraint Model from :{}".format(
os.path.join(args.weight_dir, args.dd_file)))
flag = False
chfile = os.path.join(args.weight_dir, args.dd_file)
# cp = torch.load(chfile)
# constraints_model.load_state_dict(cp['constraints_model'])
# if 'dd_update_freq' in cp:
# dd_update_freq = cp['dd_update_freq']
# print("New dd_update_freq:" , dd_update_freq)
if flag:
raise (
"why provide args.weight_dir? when both weight_file and dd_file are None"
)
print("Trainer")
trainer = Trainer.from_params(
model=pieces.model,
serialization_dir=serialization_dir,
iterator=pieces.iterator,
train_data=pieces.train_dataset,
validation_data=pieces.validation_dataset,
params=pieces.params,
validation_iterator=pieces.validation_iterator)
if args.weight_dir is not None and args.training_state_file is not None:
logging.info("Loading Training state from :{}".format(
os.path.join(args.weight_dir, args.training_state_file)))
training_state = torch.load(
os.path.join(args.weight_dir, args.training_state_file))
trainer.optimizer.load_state_dict(training_state["optimizer"])
params.assert_empty('base train command')
try:
#if backprop_after_xbatches == 1:
# print("Training setup")
# semi_trainer= gan_trainer.SemiSupervisedTrainer(trainer, constraints_model, dd_optimizer, pieces.validation_iterator, pieces.unlabelled_dataset, semi_supervision, which_mixer, dd_warmup_iters, dd_update_freq, constraints_wt, calc_valid_freq)
# print("Training start")
# metrics = semi_trainer.custom_train()
#else:
print("Training setup")
semi_trainer = gan_trainer.SemiSupervisedTrainer(
trainer,
constraints_model,
dd_optimizer,
pieces.validation_iterator,
pieces.unlabelled_dataset,
semi_supervision,
which_mixer,
dd_warmup_iters,
dd_update_freq,
constraints_wt,
calc_valid_freq,
backprop_after_xbatches,
min_pct_of_unlabelled,
dd_semi_warmup_iters,
dd_increase_freq_after,
dd_increase_freq_by,
dd_decay_lr,
args.debug,
chfile=chfile,
shuffle=args.shuffle,
dd_decay_lr_after=dd_decay_lr_after,
grad_norm_before_warmup=grad_norm_before_warmup)
print("Training start")
#print(yatin)
metrics = semi_trainer.custom_train()
except KeyboardInterrupt:
# if we have completed an epoch, try to create a model archive.
if os.path.exists(os.path.join(serialization_dir, _DEFAULT_WEIGHTS)):
logging.info(
"Training interrupted by the user. Attempting to create "
"a model archive using the current best epoch weights.")
archive_model(serialization_dir,
files_to_archive=params.files_to_archive)
raise
archive_model(serialization_dir, files_to_archive=params.files_to_archive)
common_util.dump_metrics(os.path.join(serialization_dir, "metrics.json"),
metrics,
log=True)
def _get_optimizer(self, lr: float = 1.0):
return Optimizer.from_params(self.model.named_parameters(), Params({"type": "sgd", "lr": lr}))
def train(self,
tasks,
validation_interval,
n_batches_per_pass,
weighting_method,
scaling_method,
train_params,
optimizer_params,
scheduler_params,
shared_optimizer=0,
load_model=1):
iterator = self._iterator
task_infos, metric_infos = self._setup_training(
tasks, train_params, optimizer_params, scheduler_params, iterator)
if shared_optimizer:
g_optimizer = Optimizer.from_params(
train_params, copy.deepcopy(optimizer_params))
g_scheduler = LearningRateScheduler.from_params(
g_optimizer, copy.deepcopy(scheduler_params))
else:
g_optimizer, g_scheduler = None, None
self._g_optimizer = g_optimizer
self._g_scheduler = g_scheduler
n_pass, should_stop = 0, False # define these here b/c they might get overridden on load
if self._serialization_dir is not None: # Resume from serialization path
if load_model \
and any(["model_state_epoch_" in x for x in os.listdir(self._serialization_dir)]):
n_pass, should_stop = self._restore_checkpoint()
logger.info(
"Loaded model from checkpoint. Starting at pass %d",
n_pass)
if self._grad_clipping is not None: # pylint: disable=invalid-unary-operand-type
clip_function = lambda grad: grad.clamp(-self._grad_clipping, self.
_grad_clipping)
for parameter in self._model.parameters():
if parameter.requires_grad:
parameter.register_hook(clip_function)
if weighting_method == 'uniform':
sample_weights = [1] * len(tasks)
elif weighting_method == 'proportional':
sample_weights = [
task_infos[task.name]['n_tr_batches'] for task in tasks
]
max_weight = max(sample_weights)
min_weight = min(sample_weights)
samples = random.choices(tasks,
weights=sample_weights,
k=validation_interval)
logger.info("Beginning training.")
all_tr_metrics = {}
while not should_stop:
self._model.train()
# randomly select a task
task = samples[n_pass % (validation_interval)]
task_info = task_infos[task.name]
if task_info['stopped']:
continue
tr_generator = task_info['tr_generator']
optimizer = g_optimizer if shared_optimizer else task_info[
'optimizer']
total_batches_trained = task_info['total_batches_trained']
n_batches_since_val = task_info['n_batches_since_val']
tr_loss = task_info['loss']
for batch in itertools.islice(tr_generator, n_batches_per_pass):
n_batches_since_val += 1
total_batches_trained += 1
optimizer.zero_grad()
output_dict = self._forward(batch,
task=task,
for_training=True)
assert "loss" in output_dict, "Model must return a dict containing a 'loss' key"
loss = output_dict["loss"] # optionally scale loss
if scaling_method == 'unit' and weighting_method == 'proportional':
loss /= task_info['n_tr_batches']
elif scaling_method == 'max' and weighting_method == 'proportional':
loss *= (max_weight / task_info['n_tr_batches'])
elif scaling_method == 'min' and weighting_method == 'proportional':
loss *= (min_weight / task_info['n_tr_batches'])
loss.backward()
tr_loss += loss.data.cpu().numpy()
# Gradient regularization and application
if self._grad_norm:
clip_grad_norm(self._model.parameters(), self._grad_norm)
optimizer.step()
n_pass += 1 # update per batch
# Update training progress on that task
task_info['n_batches_since_val'] = n_batches_since_val
task_info['total_batches_trained'] = total_batches_trained
task_info['loss'] = tr_loss
# Intermediate logging
if time.time() - task_info['last_log'] > self._log_interval:
task_metrics = task.get_metrics()
task_metrics["%s_loss" %
task.name] = tr_loss / n_batches_since_val
description = self._description_from_metrics(task_metrics)
logger.info("Update %d: task %s, batch %d (%d): %s", n_pass,
task.name, n_batches_since_val,
total_batches_trained, description)
task_info['last_log'] = time.time()
# Validation
if n_pass % (validation_interval) == 0:
epoch = int(n_pass / validation_interval)
logger.info("***** Pass %d / Epoch %d *****", n_pass, epoch)
# Get metrics for all training progress so far
for task in tasks:
task_info = task_infos[task.name]
n_batches_since_val = task_info['n_batches_since_val']
if n_batches_since_val > 0:
task_metrics = task.get_metrics(reset=True)
for name, value in task_metrics.items():
all_tr_metrics["%s_%s" % (task.name, name)] = value
all_tr_metrics["%s_loss" % task.name] = \
float(task_info['loss'] / n_batches_since_val)
else:
all_tr_metrics["%s_loss" % task.name] = 0.0
logger.info(
"%s: trained on %d batches, %.3f epochs", task.name,
n_batches_since_val,
n_batches_since_val / task_info['n_tr_batches'])
# Validate
logger.info("Validating...")
all_val_metrics, should_save, task_infos, metric_infos = \
self._validate(epoch, tasks, task_infos, metric_infos, iterator, g_scheduler)
# Check stopping conditions
should_stop, task_infos, metric_infos = \
self._check_stop(epoch, tasks, task_infos, metric_infos, g_optimizer)
# Log results
for name, value in all_val_metrics.items():
logger.info("Statistic: %s", name)
if name in all_tr_metrics:
logger.info("\ttraining: %3f", all_tr_metrics[name])
logger.info("\tvalidation: %3f", value)
self._metric_infos = metric_infos
self._task_infos = task_infos
all_tr_metrics = {}
samples = random.choices(tasks,
weights=sample_weights,
k=validation_interval)
if should_save:
self._save_checkpoint({
"epoch": epoch,
"should_stop": should_stop
})
logging.info('Stopped training after %d validation checks',
n_pass / validation_interval)
return self._aggregate_results(tasks, task_infos,
metric_infos) #, validation_interval)
def _get_optimizer(self, lr: float = 1.0):
optimizer_params = Params({"type": "sgd", "lr": lr})
optimizer_params["parameter_groups"] = [[[f"^{m}"], {}] for m in self.model._modules]
return Optimizer.from_params(self.model.named_parameters(), optimizer_params)
def from_params(
cls, # type: ignore
model: Model,
serialization_dir: str,
iterator: DataIterator,
train_data: Iterable[Instance],
validation_data: Optional[Iterable[Instance]],
params: Params,
validation_iterator: DataIterator = None) -> 'Trainer':
# pylint: disable=arguments-differ
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
if isinstance(cuda_device, list):
model_device = cuda_device[0]
else:
model_device = cuda_device
if model_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(model_device)
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if lr_scheduler_params:
scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params)
else:
scheduler = None
num_serialized_models_to_keep = params.pop_int(
"num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool(
"should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate",
False)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
params.assert_empty(cls.__name__)
return cls(
model,
optimizer,
iterator,
train_data,
validation_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=scheduler,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=
keep_serialized_model_every_num_seconds,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period)
def test_no_metric_wrapper_can_support_none_for_metrics(self):
model = torch.nn.Sequential(torch.nn.Linear(10, 10))
lrs = LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(),
Params({"type": "adam"})),
Params({"type": "step", "step_size": 1}))
lrs.step(None, None)
def test_reduce_on_plateau_works_when_metrics_exist(self):
model = torch.nn.Sequential(torch.nn.Linear(10, 10))
LearningRateScheduler.from_params(
Optimizer.from_params(model.named_parameters(),
Params({"type": "adam"})),
Params({"type": "reduce_on_plateau"})).step(10, None)
def from_params(cls,
model: Model,
serialization_dir: str,
iterator: DataIterator,
train_data: Iterable[Instance],
validation_data: Optional[Iterable[Instance]],
params: Params,
validation_iterator: DataIterator = None) -> 'Trainer':
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = params.pop_int("cuda_device", -1)
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
if cuda_device >= 0:
model = model.cuda(cuda_device)
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if lr_scheduler_params:
scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params)
else:
scheduler = None
num_serialized_models_to_keep = params.pop_int(
"num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
params.assert_empty(cls.__name__)
return Trainer(
model,
optimizer,
iterator,
train_data,
validation_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=scheduler,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=
keep_serialized_model_every_num_seconds,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval)
def from_params( # type: ignore
cls,
params: Params,
serialization_dir: str,
recover: bool = False,
cache_directory: str = None,
cache_prefix: str = None,
) -> "CallbackTrainer":
pieces = TrainerPieces.from_params(params, serialization_dir, recover,
cache_directory, cache_prefix)
model = pieces.model
params = pieces.params
validation_iterator = pieces.validation_iterator or pieces.iterator
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
check_for_gpu(cuda_device)
if cuda_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(cuda_device)
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
callbacks_params = params.pop("callbacks", [])
callbacks: List[Callback] = [
Callback.from_params(
params=callback_params,
model=model,
optimizer=optimizer,
instances=pieces.train_dataset,
iterator=pieces.iterator,
shuffle=shuffle,
validation_data=pieces.validation_dataset,
validation_iterator=validation_iterator,
serialization_dir=serialization_dir,
) for callback_params in callbacks_params
]
distributed = params.pop_bool("distributed", False)
world_size = params.pop_int("world_size", 1)
if distributed:
rank = cuda_device
else:
rank = 0
params.assert_empty(cls.__name__)
return cls(
model,
pieces.train_dataset,
pieces.iterator,
optimizer,
num_epochs=num_epochs,
shuffle=shuffle,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
callbacks=callbacks,
distributed=distributed,
rank=rank,
world_size=world_size,
)
def _from_params(
cls, # type: ignore
model: Model,
serialization_dir: str,
iterator: DataIterator,
train_data: Iterable[Instance],
validation_data: Optional[Iterable[Instance]],
params: Params,
validation_iterator: DataIterator = None) -> DecompTrainer:
# pylint: disable=arguments-differ
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
momentum_scheduler_params = params.pop("momentum_scheduler", None)
validation_data_path = params.pop("validation_data_path", None)
validation_prediction_path = params.pop("validation_prediction_path", None)
semantics_only = params.pop("semantics_only", False)
drop_syntax = params.pop("drop_syntax", True)
include_attribute_scores = params.pop("include_attribute_scores", False)
warmup_epochs = params.pop("warmup_epochs", 0)
if isinstance(cuda_device, list):
model_device = cuda_device[0]
else:
model_device = cuda_device
if model_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(model_device)
bert_optim_params = params.pop("bert_optimizer", None)
bert_name = "_bert_encoder"
if bert_optim_params is not None:
tune_after_layer_num = params.pop("bert_tune_layer", 12)
frozen_regex_str = [
"(_bert_encoder\.bert_model\.embeddings.*)",
"(_bert_encoder\.bert_model\.pooler.*)"
]
tune_regex_str = []
for i in range(0, 12):
# match all numbers greater than layer num via disjunction
tune_regex_one = f"({bert_name}\.bert_model\.encoder\.layer\.{i}\..*)"
if i >= tune_after_layer_num:
tune_regex_str.append(tune_regex_one)
else:
frozen_regex_str.append(tune_regex_one)
tune_regex = re.compile("|".join(tune_regex_str))
frozen_regex = re.compile("|".join(frozen_regex_str))
# decide which params require grad for which optimizer
all_names = [n for n, p in model.named_parameters()]
tune_bert_names = [
n for n in all_names if tune_regex.match(n) is not None
]
frozen_names = [
n for n in all_names if frozen_regex.match(n) is not None
]
# assert that they're disjoint
assert (len(set(frozen_names) & set(tune_bert_names)) == 0)
# set tunable params to require gradient, frozen ones to not require
for i, (n, p) in enumerate(model.named_parameters()):
if n in frozen_names:
p.requires_grad = False
else:
p.requires_grad = True
# extract BERT
bert_params = [[n, p] for n, p in model.named_parameters()
if p.requires_grad and n in tune_bert_names]
# make sure this matches the tuneable bert params
assert ([x[0] for x in bert_params] == tune_bert_names)
bert_optimizer = Optimizer.from_params(bert_params, bert_optim_params)
else:
# freeze all BERT params
tune_bert_names = []
bert_optimizer = None
for i, (n, p) in enumerate(model.named_parameters()):
if "_bert_encoder" in n:
p.requires_grad = False
# model params
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad and n not in tune_bert_names]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if "moving_average" in params:
moving_average = MovingAverage.from_params(
params.pop("moving_average"), parameters=parameters)
else:
moving_average = None
if lr_scheduler_params:
lr_scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params)
else:
lr_scheduler = None
if momentum_scheduler_params:
momentum_scheduler = MomentumScheduler.from_params(
optimizer, momentum_scheduler_params)
else:
momentum_scheduler = None
if 'checkpointer' in params:
if 'keep_serialized_model_every_num_seconds' in params or \
'num_serialized_models_to_keep' in params:
raise ConfigurationError(
"Checkpointer may be initialized either from the 'checkpointer' key or from the "
"keys 'num_serialized_models_to_keep' and 'keep_serialized_model_every_num_seconds'"
" but the passed config uses both methods.")
checkpointer = Checkpointer.from_params(params.pop("checkpointer"))
else:
num_serialized_models_to_keep = params.pop_int(
"num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
checkpointer = Checkpointer(
serialization_dir=serialization_dir,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=
keep_serialized_model_every_num_seconds)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool(
"should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate",
False)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
syntactic_method = params.pop("syntactic_method", None)
accumulate_batches = params.pop("accumulate_batches", 1)
params.assert_empty(cls.__name__)
return cls(model=model,
optimizer=optimizer,
bert_optimizer=bert_optimizer,
iterator=iterator,
train_dataset=train_data,
validation_dataset=validation_data,
validation_data_path=validation_data_path,
validation_prediction_path=validation_prediction_path,
semantics_only=semantics_only,
warmup_epochs=warmup_epochs,
syntactic_method=syntactic_method,
drop_syntax=drop_syntax,
include_attribute_scores=include_attribute_scores,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=lr_scheduler,
momentum_scheduler=momentum_scheduler,
checkpointer=checkpointer,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period,
moving_average=moving_average,
accumulate_batches=accumulate_batches)
def from_params(
cls, # type: ignore
params: Params,
serialization_dir: str,
recover: bool = False,
cache_directory: str = None,
cache_prefix: str = None) -> 'Trainer':
# pylint: disable=arguments-differ
# We have to call TrainerPieces.from_params since we are using our own Trainer
pieces = TrainerPieces.from_params(params, serialization_dir, recover)
model = pieces.model
serialization_dir = serialization_dir
iterator = pieces.iterator
train_data = pieces.train_dataset
validation_data = pieces.validation_dataset
validation_iterator = pieces.validation_iterator
params = pieces.params
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
accumulation_steps = params.pop("accumulation_steps", 0)
opt_level = params.pop("opt_level", "O1")
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
momentum_scheduler_params = params.pop("momentum_scheduler", None)
half_precision = params.pop("half_precision", False)
warmup_proportion = params.pop("warmup_proportion", None)
pretrained_model = params.pop("pretrained_model", None)
if pretrained_model:
logger.info('Loading pretrained model from', pretrained_model)
model = load_archive(pretrained_model).model
model._discriminative_loss_weight = 1 # TODO: fix this hack
if isinstance(cuda_device, list):
model_device = cuda_device[0]
else:
model_device = cuda_device
if model_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(model_device)
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if "moving_average" in params:
moving_average = MovingAverage.from_params(
params.pop("moving_average"), parameters=parameters)
else:
moving_average = None
if lr_scheduler_params:
lr_scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params)
else:
lr_scheduler = None
if momentum_scheduler_params:
momentum_scheduler = MomentumScheduler.from_params(
optimizer, momentum_scheduler_params)
else:
momentum_scheduler = None
if 'checkpointer' in params:
if 'keep_serialized_model_every_num_seconds' in params or \
'num_serialized_models_to_keep' in params:
raise ConfigurationError(
"Checkpointer may be initialized either from the 'checkpointer' key or from the "
"keys 'num_serialized_models_to_keep' and 'keep_serialized_model_every_num_seconds'"
" but the passed config uses both methods.")
checkpointer = Checkpointer.from_params(params.pop("checkpointer"))
else:
num_serialized_models_to_keep = params.pop_int(
"num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
checkpointer = Checkpointer(
serialization_dir=serialization_dir,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=
keep_serialized_model_every_num_seconds)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool(
"should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate",
False)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
params.assert_empty(cls.__name__)
return cls(
model,
optimizer,
iterator,
train_data,
validation_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
accumulation_steps=accumulation_steps,
opt_level=opt_level,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=lr_scheduler,
momentum_scheduler=momentum_scheduler,
checkpointer=checkpointer,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period,
moving_average=moving_average,
half_precision=half_precision,
warmup_proportion=warmup_proportion)
def from_params( # type: ignore
cls,
model: Model,
serialization_dir: str,
iterator: DataIterator,
train_data: Iterable[Instance],
validation_data: Optional[Iterable[Instance]],
params: Params,
validation_iterator: DataIterator = None,
) -> "Trainer":
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
momentum_scheduler_params = params.pop("momentum_scheduler", None)
if isinstance(cuda_device, list):
model_device = cuda_device[0]
else:
model_device = cuda_device
if model_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(model_device)
parameters = [[n, p] for n, p in model.named_parameters()
if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if "moving_average" in params:
moving_average = MovingAverage.from_params(
params.pop("moving_average"), parameters=parameters)
else:
moving_average = None
if lr_scheduler_params:
lr_scheduler = LearningRateScheduler.from_params(
optimizer, lr_scheduler_params)
else:
lr_scheduler = None
if momentum_scheduler_params:
momentum_scheduler = MomentumScheduler.from_params(
optimizer, momentum_scheduler_params)
else:
momentum_scheduler = None
if "checkpointer" in params:
if "keep_serialized_model_every_num_seconds" in params \
or "num_serialized_models_to_keep" in params:
raise ConfigurationError(
"Checkpointer may be initialized either from the 'checkpointer' key or from the "
"keys 'num_serialized_models_to_keep' and 'keep_serialized_model_every_num_seconds'"
" but the passed config uses both methods.")
checkpointer = Checkpointer.from_params(params.pop("checkpointer"))
else:
num_serialized_models_to_keep = params.pop_int(
"num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
checkpointer = Checkpointer(
serialization_dir=serialization_dir,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=
keep_serialized_model_every_num_seconds,
)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool(
"should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate",
False)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
params.assert_empty(cls.__name__)
return cls(
model,
optimizer,
iterator,
train_data,
validation_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=lr_scheduler,
momentum_scheduler=momentum_scheduler,
checkpointer=checkpointer,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period,
moving_average=moving_average,
)
def test_no_metric_wrapper_can_support_none_for_metrics(self):
model = torch.nn.Sequential(torch.nn.Linear(10, 10))
lrs = LearningRateScheduler.from_params(Optimizer.from_params(model.named_parameters(),
Params({"type": "adam"})),
Params({"type": "step", "step_size": 1}))
lrs.step(None, None)