def test_mismatched_dimensions_raise_configuration_errors(self):
params = Params.from_file(self.param_file)
# Make the input_dim to the first feedforward_layer wrong - it should be 2.
params["model"]["attend_feedforward"]["input_dim"] = 10
with pytest.raises(ConfigurationError):
Model.from_params(vocab=self.vocab, params=params.pop("model"))
params = Params.from_file(self.param_file)
# Make the projection output_dim of the last layer wrong - it should be
# 3, equal to the number of classes.
params["model"]["aggregate_feedforward"]["output_dim"] = 10
with pytest.raises(ConfigurationError):
Model.from_params(vocab=self.vocab, params=params.pop("model"))
def train_model_from_file(parameter_filename: str,
serialization_dir: str,
overrides: str = "",
file_friendly_logging: bool = False,
recover: bool = False) -> Model:
"""
A wrapper around :func:`train_model` which loads the params from a file.
Parameters
----------
param_path : ``str``
A json parameter file specifying an AllenNLP experiment.
serialization_dir : ``str``
The directory in which to save results and logs. We just pass this along to
:func:`train_model`.
overrides : ``str``
A JSON string that we will use to override values in the input parameter file.
file_friendly_logging : ``bool``, optional (default=False)
If ``True``, we make our output more friendly to saved model files. We just pass this
along to :func:`train_model`.
recover : ``bool`, optional (default=False)
If ``True``, we will try to recover a training run from an existing serialization
directory. This is only intended for use when something actually crashed during the middle
of a run. For continuing training a model on new data, see the ``fine-tune`` command.
"""
# Load the experiment config from a file and pass it to ``train_model``.
params = Params.from_file(parameter_filename, overrides)
return train_model(params, serialization_dir, file_friendly_logging,
recover)
def fine_tune_model_from_file_paths(model_archive_path: str,
config_file: str,
serialization_dir: str,
overrides: str = "",
extend_vocab: bool = False,
file_friendly_logging: bool = False) -> Model:
"""
A wrapper around :func:`fine_tune_model` which loads the model archive from a file.
Parameters
----------
model_archive_path : ``str``
Path to a saved model archive that is the result of running the ``train`` command.
config_file : ``str``
A configuration file specifying how to continue training. The format is identical to the
configuration file for the ``train`` command, but any contents in the ``model`` section is
ignored (as we are using the provided model archive instead).
serialization_dir : ``str``
The directory in which to save results and logs. We just pass this along to
:func:`fine_tune_model`.
overrides : ``str``
A JSON string that we will use to override values in the input parameter file.
file_friendly_logging : ``bool``, optional (default=False)
If ``True``, we make our output more friendly to saved model files. We just pass this
along to :func:`fine_tune_model`.
"""
# We don't need to pass in `cuda_device` here, because the trainer will call `model.cuda()` if
# necessary.
archive = load_archive(model_archive_path)
params = Params.from_file(config_file, overrides)
return fine_tune_model(model=archive.model,
params=params,
serialization_dir=serialization_dir,
extend_vocab=extend_vocab,
file_friendly_logging=file_friendly_logging)
def test_batch_predictions_are_consistent(self):
# The CNN encoder has problems with this kind of test - it's not properly masked yet, so
# changing the amount of padding in the batch will result in small differences in the
# output of the encoder. Because BiDAF is so deep, these differences get magnified through
# the network and make this test impossible. So, we'll remove the CNN encoder entirely
# from the model for this test. If/when we fix the CNN encoder to work correctly with
# masking, we can change this back to how the other models run this test, with just a
# single line.
# pylint: disable=protected-access,attribute-defined-outside-init
# Save some state.
saved_model = self.model
saved_instances = self.instances
# Modify the state, run the test with modified state.
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
reader._token_indexers = {'tokens': reader._token_indexers['tokens']}
self.instances = reader.read(self.FIXTURES_ROOT / 'data' /
'squad.json')
vocab = Vocabulary.from_instances(self.instances)
for instance in self.instances:
instance.index_fields(vocab)
del params['model']['text_field_embedder']['token_embedders'][
'token_characters']
params['model']['phrase_layer']['input_size'] = 2
self.model = Model.from_params(vocab=vocab, params=params['model'])
self.ensure_batch_predictions_are_consistent()
# Restore the state.
self.model = saved_model
self.instances = saved_instances
def test_model_load(self):
params = Params.from_file(self.FIXTURES_ROOT /
'decomposable_attention' / 'experiment.json')
model = Model.load(params,
serialization_dir=self.FIXTURES_ROOT /
'decomposable_attention' / 'serialization')
assert isinstance(model, DecomposableAttention)
def test_mismatching_dimensions_throws_configuration_error(self):
params = Params.from_file(self.param_file)
# Make the phrase layer wrong - it should be 10 to match
# the embedding + char cnn dimensions.
params["model"]["phrase_layer"]["input_size"] = 12
with pytest.raises(ConfigurationError):
Model.from_params(vocab=self.vocab, params=params.pop("model"))
params = Params.from_file(self.param_file)
# Make the modeling layer input_dimension wrong - it should be 40 to match
# 4 * output_dim of the phrase_layer.
params["model"]["phrase_layer"]["input_size"] = 30
with pytest.raises(ConfigurationError):
Model.from_params(vocab=self.vocab, params=params.pop("model"))
params = Params.from_file(self.param_file)
# Make the modeling layer input_dimension wrong - it should be 70 to match
# 4 * phrase_layer.output_dim + 3 * modeling_layer.output_dim.
params["model"]["span_end_encoder"]["input_size"] = 50
with pytest.raises(ConfigurationError):
Model.from_params(vocab=self.vocab, params=params.pop("model"))
def test_forward_with_epoch_num_changes_cost_weight(self):
# Redefining model. We do not want this to change the state of ``self.model``.
params = Params.from_file(self.param_file)
model = Model.from_params(vocab=self.vocab, params=params['model'])
# Initial cost weight, before forward is called.
assert model._checklist_cost_weight == 0.8
iterator = EpochTrackingBucketIterator(sorting_keys=[['sentence', 'num_tokens']])
cost_weights = []
for epoch_data in iterator(self.dataset, num_epochs=4):
model.forward(**epoch_data)
cost_weights.append(model._checklist_cost_weight)
# The config file has ``wait_num_epochs`` set to 0, so the model starts decreasing the cost
# weight at epoch 0 itself.
assert_almost_equal(cost_weights, [0.72, 0.648, 0.5832, 0.52488])
def test_embed_actions_works_with_batched_and_padded_input(self):
params = Params.from_file(self.param_file)
model = Model.from_params(vocab=self.vocab, params=params['model'])
action_embedding_weights = model._action_embedder.weight
rule1 = model.vocab.get_token_from_index(1, 'rule_labels')
rule1_tensor = torch.LongTensor([1])
rule2 = model.vocab.get_token_from_index(2, 'rule_labels')
rule2_tensor = torch.LongTensor([2])
rule3 = model.vocab.get_token_from_index(3, 'rule_labels')
rule3_tensor = torch.LongTensor([3])
actions = [
[
(rule1, True, rule1_tensor),
(rule2, True, rule2_tensor),
# This one is padding; the tensors shouldn't matter here.
('', False, None)
],
[(rule3, True, rule3_tensor), ('instance_action', False, None),
(rule1, True, rule1_tensor)]
]
embedded_actions, _, _, action_indices = model._embed_actions(actions)
assert action_indices[(0, 0)] == action_indices[(1, 2)]
assert action_indices[(1, 1)] == -1
assert len(set(action_indices.values())) == 4
# Now we'll go through all three unique actions and make sure the embedding is as we expect.
action_embedding = embedded_actions[action_indices[(0, 0)]]
expected_action_embedding = action_embedding_weights[action_indices[(
0, 0)]]
assert_almost_equal(action_embedding.cpu().data.numpy(),
expected_action_embedding.cpu().data.numpy())
action_embedding = embedded_actions[action_indices[(0, 1)]]
expected_action_embedding = action_embedding_weights[action_indices[(
0, 1)]]
assert_almost_equal(action_embedding.cpu().data.numpy(),
expected_action_embedding.cpu().data.numpy())
action_embedding = embedded_actions[action_indices[(1, 0)]]
expected_action_embedding = action_embedding_weights[action_indices[(
1, 0)]]
assert_almost_equal(action_embedding.cpu().data.numpy(),
expected_action_embedding.cpu().data.numpy())
def test_file_archiving(self):
# This happens to be a good place to test auxiliary file archiving.
# Train the model
params = Params.from_file(self.FIXTURES_ROOT / 'elmo' / 'config' /
'characters_token_embedder.json')
serialization_dir = os.path.join(self.TEST_DIR, 'serialization')
train_model(params, serialization_dir)
# Inspect the archive
archive_file = os.path.join(serialization_dir, 'model.tar.gz')
unarchive_dir = os.path.join(self.TEST_DIR, 'unarchive')
with tarfile.open(archive_file, 'r:gz') as archive:
archive.extractall(unarchive_dir)
# It should contain `files_to_archive.json`
fta_file = os.path.join(unarchive_dir, 'files_to_archive.json')
assert os.path.exists(fta_file)
# Which should properly contain { flattened_key -> original_filename }
with open(fta_file) as fta:
files_to_archive = json.loads(fta.read())
assert files_to_archive == {
'model.text_field_embedder.token_embedders.elmo.options_file':
str(
pathlib.Path('allennlp') / 'tests' / 'fixtures' / 'elmo' /
'options.json'),
'model.text_field_embedder.token_embedders.elmo.weight_file':
str(
pathlib.Path('allennlp') / 'tests' / 'fixtures' / 'elmo' /
'lm_weights.hdf5'),
}
# Check that the unarchived contents of those files match the original contents.
for key, original_filename in files_to_archive.items():
new_filename = os.path.join(unarchive_dir, "fta", key)
assert filecmp.cmp(original_filename, new_filename)
def set_up_model(self, param_file, dataset_file):
# pylint: disable=attribute-defined-outside-init
self.param_file = param_file
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
instances = reader.read(dataset_file)
# Use parameters for vocabulary if they are present in the config file, so that choices like
# "non_padded_namespaces", "min_count" etc. can be set if needed.
if 'vocabulary' in params:
vocab_params = params['vocabulary']
vocab = Vocabulary.from_params(params=vocab_params,
instances=instances)
else:
vocab = Vocabulary.from_instances(instances)
self.vocab = vocab
self.instances = instances
self.model = Model.from_params(vocab=self.vocab,
params=params['model'])
# TODO(joelgrus) get rid of these
# (a lot of the model tests use them, so they'll have to be changed)
self.dataset = Batch(self.instances)
self.dataset.index_instances(self.vocab)
def create_serialization_dir(params: Params, serialization_dir: str,
recover: bool) -> None:
"""
This function creates the serialization directory if it doesn't exist. If it already exists
and is non-empty, then it verifies that we're recovering from a training with an identical configuration.
Parameters
----------
params: ``Params``
A parameter object specifying an AllenNLP Experiment.
serialization_dir: ``str``
The directory in which to save results and logs.
recover: ``bool``
If ``True``, we will try to recover from an existing serialization directory, and crash if
the directory doesn't exist, or doesn't match the configuration we're given.
"""
if os.path.exists(serialization_dir) and os.listdir(serialization_dir):
if not recover:
raise ConfigurationError(
f"Serialization directory ({serialization_dir}) already exists and is "
f"not empty. Specify --recover to recover training from existing output."
)
logger.info(f"Recovering from prior training at {serialization_dir}.")
recovered_config_file = os.path.join(serialization_dir, CONFIG_NAME)
if not os.path.exists(recovered_config_file):
raise ConfigurationError(
"The serialization directory already exists but doesn't "
"contain a config.json. You probably gave the wrong directory."
)
else:
loaded_params = Params.from_file(recovered_config_file)
# Check whether any of the training configuration differs from the configuration we are
# resuming. If so, warn the user that training may fail.
fail = False
flat_params = params.as_flat_dict()
flat_loaded = loaded_params.as_flat_dict()
for key in flat_params.keys() - flat_loaded.keys():
logger.error(
f"Key '{key}' found in training configuration but not in the serialization "
f"directory we're recovering from.")
fail = True
for key in flat_loaded.keys() - flat_params.keys():
logger.error(
f"Key '{key}' found in the serialization directory we're recovering from "
f"but not in the training config.")
fail = True
for key in flat_params.keys():
if flat_params.get(key, None) != flat_loaded.get(key, None):
logger.error(
f"Value for '{key}' in training configuration does not match that the value in "
f"the serialization directory we're recovering from: "
f"{flat_params[key]} != {flat_loaded[key]}")
fail = True
if fail:
raise ConfigurationError(
"Training configuration does not match the configuration we're "
"recovering from.")
else:
if recover:
raise ConfigurationError(
f"--recover specified but serialization_dir ({serialization_dir}) "
"does not exist. There is nothing to recover from.")
os.makedirs(serialization_dir, exist_ok=True)
def ensure_model_can_train_save_and_load(self,
param_file: str,
tolerance: float = 1e-4,
cuda_device: int = -1):
save_dir = self.TEST_DIR / "save_and_load_test"
archive_file = save_dir / "model.tar.gz"
model = train_model_from_file(param_file, save_dir)
loaded_model = load_archive(archive_file,
cuda_device=cuda_device).model
state_keys = model.state_dict().keys()
loaded_state_keys = loaded_model.state_dict().keys()
assert state_keys == loaded_state_keys
# First we make sure that the state dict (the parameters) are the same for both models.
for key in state_keys:
assert_allclose(model.state_dict()[key].cpu().numpy(),
loaded_model.state_dict()[key].cpu().numpy(),
err_msg=key)
params = Params.from_file(param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
# Need to duplicate params because Iterator.from_params will consume.
iterator_params = params['iterator']
iterator_params2 = Params(copy.deepcopy(iterator_params.as_dict()))
iterator = DataIterator.from_params(iterator_params)
iterator2 = DataIterator.from_params(iterator_params2)
# We'll check that even if we index the dataset with each model separately, we still get
# the same result out.
model_dataset = reader.read(params['validation_data_path'])
iterator.index_with(model.vocab)
model_batch = next(
iterator(model_dataset, shuffle=False, cuda_device=cuda_device))
loaded_dataset = reader.read(params['validation_data_path'])
iterator2.index_with(loaded_model.vocab)
loaded_batch = next(
iterator2(loaded_dataset, shuffle=False, cuda_device=cuda_device))
# Check gradients are None for non-trainable parameters and check that
# trainable parameters receive some gradient if they are trainable.
self.check_model_computes_gradients_correctly(model, model_batch)
# The datasets themselves should be identical.
assert model_batch.keys() == loaded_batch.keys()
for key in model_batch.keys():
self.assert_fields_equal(model_batch[key], loaded_batch[key], key,
1e-6)
# Set eval mode, to turn off things like dropout, then get predictions.
model.eval()
loaded_model.eval()
# Models with stateful RNNs need their states reset to have consistent
# behavior after loading.
for model_ in [model, loaded_model]:
for module in model_.modules():
if hasattr(module, 'stateful') and module.stateful:
module.reset_states()
model_predictions = model(**model_batch)
loaded_model_predictions = loaded_model(**loaded_batch)
# Check loaded model's loss exists and we can compute gradients, for continuing training.
loaded_model_loss = loaded_model_predictions["loss"]
assert loaded_model_loss is not None
loaded_model_loss.backward()
# Both outputs should have the same keys and the values for these keys should be close.
for key in model_predictions.keys():
self.assert_fields_equal(model_predictions[key],
loaded_model_predictions[key],
name=key,
tolerance=tolerance)
return model, loaded_model
