def test_mismatched_dimensions_raise_configuration_errors(self): params = Params.from_file(self.param_file) # Make the input_dim to the first feedforward_layer wrong - it should be 2. params["model"]["attend_feedforward"]["input_dim"] = 10 with pytest.raises(ConfigurationError): Model.from_params(vocab=self.vocab, params=params.pop("model")) params = Params.from_file(self.param_file) # Make the projection output_dim of the last layer wrong - it should be # 3, equal to the number of classes. params["model"]["aggregate_feedforward"]["output_dim"] = 10 with pytest.raises(ConfigurationError): Model.from_params(vocab=self.vocab, params=params.pop("model"))
def train_model_from_file(parameter_filename: str, serialization_dir: str, overrides: str = "", file_friendly_logging: bool = False, recover: bool = False) -> Model: """ A wrapper around :func:`train_model` which loads the params from a file. Parameters ---------- param_path : ``str`` A json parameter file specifying an AllenNLP experiment. serialization_dir : ``str`` The directory in which to save results and logs. We just pass this along to :func:`train_model`. overrides : ``str`` A JSON string that we will use to override values in the input parameter file. file_friendly_logging : ``bool``, optional (default=False) If ``True``, we make our output more friendly to saved model files. We just pass this along to :func:`train_model`. recover : ``bool`, optional (default=False) If ``True``, we will try to recover a training run from an existing serialization directory. This is only intended for use when something actually crashed during the middle of a run. For continuing training a model on new data, see the ``fine-tune`` command. """ # Load the experiment config from a file and pass it to ``train_model``. params = Params.from_file(parameter_filename, overrides) return train_model(params, serialization_dir, file_friendly_logging, recover)
def fine_tune_model_from_file_paths(model_archive_path: str, config_file: str, serialization_dir: str, overrides: str = "", extend_vocab: bool = False, file_friendly_logging: bool = False) -> Model: """ A wrapper around :func:`fine_tune_model` which loads the model archive from a file. Parameters ---------- model_archive_path : ``str`` Path to a saved model archive that is the result of running the ``train`` command. config_file : ``str`` A configuration file specifying how to continue training. The format is identical to the configuration file for the ``train`` command, but any contents in the ``model`` section is ignored (as we are using the provided model archive instead). serialization_dir : ``str`` The directory in which to save results and logs. We just pass this along to :func:`fine_tune_model`. overrides : ``str`` A JSON string that we will use to override values in the input parameter file. file_friendly_logging : ``bool``, optional (default=False) If ``True``, we make our output more friendly to saved model files. We just pass this along to :func:`fine_tune_model`. """ # We don't need to pass in `cuda_device` here, because the trainer will call `model.cuda()` if # necessary. archive = load_archive(model_archive_path) params = Params.from_file(config_file, overrides) return fine_tune_model(model=archive.model, params=params, serialization_dir=serialization_dir, extend_vocab=extend_vocab, file_friendly_logging=file_friendly_logging)
def test_batch_predictions_are_consistent(self): # The CNN encoder has problems with this kind of test - it's not properly masked yet, so # changing the amount of padding in the batch will result in small differences in the # output of the encoder. Because BiDAF is so deep, these differences get magnified through # the network and make this test impossible. So, we'll remove the CNN encoder entirely # from the model for this test. If/when we fix the CNN encoder to work correctly with # masking, we can change this back to how the other models run this test, with just a # single line. # pylint: disable=protected-access,attribute-defined-outside-init # Save some state. saved_model = self.model saved_instances = self.instances # Modify the state, run the test with modified state. params = Params.from_file(self.param_file) reader = DatasetReader.from_params(params['dataset_reader']) reader._token_indexers = {'tokens': reader._token_indexers['tokens']} self.instances = reader.read(self.FIXTURES_ROOT / 'data' / 'squad.json') vocab = Vocabulary.from_instances(self.instances) for instance in self.instances: instance.index_fields(vocab) del params['model']['text_field_embedder']['token_embedders'][ 'token_characters'] params['model']['phrase_layer']['input_size'] = 2 self.model = Model.from_params(vocab=vocab, params=params['model']) self.ensure_batch_predictions_are_consistent() # Restore the state. self.model = saved_model self.instances = saved_instances
def test_model_load(self): params = Params.from_file(self.FIXTURES_ROOT / 'decomposable_attention' / 'experiment.json') model = Model.load(params, serialization_dir=self.FIXTURES_ROOT / 'decomposable_attention' / 'serialization') assert isinstance(model, DecomposableAttention)
def test_mismatching_dimensions_throws_configuration_error(self): params = Params.from_file(self.param_file) # Make the phrase layer wrong - it should be 10 to match # the embedding + char cnn dimensions. params["model"]["phrase_layer"]["input_size"] = 12 with pytest.raises(ConfigurationError): Model.from_params(vocab=self.vocab, params=params.pop("model")) params = Params.from_file(self.param_file) # Make the modeling layer input_dimension wrong - it should be 40 to match # 4 * output_dim of the phrase_layer. params["model"]["phrase_layer"]["input_size"] = 30 with pytest.raises(ConfigurationError): Model.from_params(vocab=self.vocab, params=params.pop("model")) params = Params.from_file(self.param_file) # Make the modeling layer input_dimension wrong - it should be 70 to match # 4 * phrase_layer.output_dim + 3 * modeling_layer.output_dim. params["model"]["span_end_encoder"]["input_size"] = 50 with pytest.raises(ConfigurationError): Model.from_params(vocab=self.vocab, params=params.pop("model"))
def test_forward_with_epoch_num_changes_cost_weight(self): # Redefining model. We do not want this to change the state of ``self.model``. params = Params.from_file(self.param_file) model = Model.from_params(vocab=self.vocab, params=params['model']) # Initial cost weight, before forward is called. assert model._checklist_cost_weight == 0.8 iterator = EpochTrackingBucketIterator(sorting_keys=[['sentence', 'num_tokens']]) cost_weights = [] for epoch_data in iterator(self.dataset, num_epochs=4): model.forward(**epoch_data) cost_weights.append(model._checklist_cost_weight) # The config file has ``wait_num_epochs`` set to 0, so the model starts decreasing the cost # weight at epoch 0 itself. assert_almost_equal(cost_weights, [0.72, 0.648, 0.5832, 0.52488])
def test_embed_actions_works_with_batched_and_padded_input(self): params = Params.from_file(self.param_file) model = Model.from_params(vocab=self.vocab, params=params['model']) action_embedding_weights = model._action_embedder.weight rule1 = model.vocab.get_token_from_index(1, 'rule_labels') rule1_tensor = torch.LongTensor([1]) rule2 = model.vocab.get_token_from_index(2, 'rule_labels') rule2_tensor = torch.LongTensor([2]) rule3 = model.vocab.get_token_from_index(3, 'rule_labels') rule3_tensor = torch.LongTensor([3]) actions = [ [ (rule1, True, rule1_tensor), (rule2, True, rule2_tensor), # This one is padding; the tensors shouldn't matter here. ('', False, None) ], [(rule3, True, rule3_tensor), ('instance_action', False, None), (rule1, True, rule1_tensor)] ] embedded_actions, _, _, action_indices = model._embed_actions(actions) assert action_indices[(0, 0)] == action_indices[(1, 2)] assert action_indices[(1, 1)] == -1 assert len(set(action_indices.values())) == 4 # Now we'll go through all three unique actions and make sure the embedding is as we expect. action_embedding = embedded_actions[action_indices[(0, 0)]] expected_action_embedding = action_embedding_weights[action_indices[( 0, 0)]] assert_almost_equal(action_embedding.cpu().data.numpy(), expected_action_embedding.cpu().data.numpy()) action_embedding = embedded_actions[action_indices[(0, 1)]] expected_action_embedding = action_embedding_weights[action_indices[( 0, 1)]] assert_almost_equal(action_embedding.cpu().data.numpy(), expected_action_embedding.cpu().data.numpy()) action_embedding = embedded_actions[action_indices[(1, 0)]] expected_action_embedding = action_embedding_weights[action_indices[( 1, 0)]] assert_almost_equal(action_embedding.cpu().data.numpy(), expected_action_embedding.cpu().data.numpy())
def test_file_archiving(self): # This happens to be a good place to test auxiliary file archiving. # Train the model params = Params.from_file(self.FIXTURES_ROOT / 'elmo' / 'config' / 'characters_token_embedder.json') serialization_dir = os.path.join(self.TEST_DIR, 'serialization') train_model(params, serialization_dir) # Inspect the archive archive_file = os.path.join(serialization_dir, 'model.tar.gz') unarchive_dir = os.path.join(self.TEST_DIR, 'unarchive') with tarfile.open(archive_file, 'r:gz') as archive: archive.extractall(unarchive_dir) # It should contain `files_to_archive.json` fta_file = os.path.join(unarchive_dir, 'files_to_archive.json') assert os.path.exists(fta_file) # Which should properly contain { flattened_key -> original_filename } with open(fta_file) as fta: files_to_archive = json.loads(fta.read()) assert files_to_archive == { 'model.text_field_embedder.token_embedders.elmo.options_file': str( pathlib.Path('allennlp') / 'tests' / 'fixtures' / 'elmo' / 'options.json'), 'model.text_field_embedder.token_embedders.elmo.weight_file': str( pathlib.Path('allennlp') / 'tests' / 'fixtures' / 'elmo' / 'lm_weights.hdf5'), } # Check that the unarchived contents of those files match the original contents. for key, original_filename in files_to_archive.items(): new_filename = os.path.join(unarchive_dir, "fta", key) assert filecmp.cmp(original_filename, new_filename)
def set_up_model(self, param_file, dataset_file): # pylint: disable=attribute-defined-outside-init self.param_file = param_file params = Params.from_file(self.param_file) reader = DatasetReader.from_params(params['dataset_reader']) instances = reader.read(dataset_file) # Use parameters for vocabulary if they are present in the config file, so that choices like # "non_padded_namespaces", "min_count" etc. can be set if needed. if 'vocabulary' in params: vocab_params = params['vocabulary'] vocab = Vocabulary.from_params(params=vocab_params, instances=instances) else: vocab = Vocabulary.from_instances(instances) self.vocab = vocab self.instances = instances self.model = Model.from_params(vocab=self.vocab, params=params['model']) # TODO(joelgrus) get rid of these # (a lot of the model tests use them, so they'll have to be changed) self.dataset = Batch(self.instances) self.dataset.index_instances(self.vocab)
def create_serialization_dir(params: Params, serialization_dir: str, recover: bool) -> None: """ This function creates the serialization directory if it doesn't exist. If it already exists and is non-empty, then it verifies that we're recovering from a training with an identical configuration. Parameters ---------- params: ``Params`` A parameter object specifying an AllenNLP Experiment. serialization_dir: ``str`` The directory in which to save results and logs. recover: ``bool`` If ``True``, we will try to recover from an existing serialization directory, and crash if the directory doesn't exist, or doesn't match the configuration we're given. """ if os.path.exists(serialization_dir) and os.listdir(serialization_dir): if not recover: raise ConfigurationError( f"Serialization directory ({serialization_dir}) already exists and is " f"not empty. Specify --recover to recover training from existing output." ) logger.info(f"Recovering from prior training at {serialization_dir}.") recovered_config_file = os.path.join(serialization_dir, CONFIG_NAME) if not os.path.exists(recovered_config_file): raise ConfigurationError( "The serialization directory already exists but doesn't " "contain a config.json. You probably gave the wrong directory." ) else: loaded_params = Params.from_file(recovered_config_file) # Check whether any of the training configuration differs from the configuration we are # resuming. If so, warn the user that training may fail. fail = False flat_params = params.as_flat_dict() flat_loaded = loaded_params.as_flat_dict() for key in flat_params.keys() - flat_loaded.keys(): logger.error( f"Key '{key}' found in training configuration but not in the serialization " f"directory we're recovering from.") fail = True for key in flat_loaded.keys() - flat_params.keys(): logger.error( f"Key '{key}' found in the serialization directory we're recovering from " f"but not in the training config.") fail = True for key in flat_params.keys(): if flat_params.get(key, None) != flat_loaded.get(key, None): logger.error( f"Value for '{key}' in training configuration does not match that the value in " f"the serialization directory we're recovering from: " f"{flat_params[key]} != {flat_loaded[key]}") fail = True if fail: raise ConfigurationError( "Training configuration does not match the configuration we're " "recovering from.") else: if recover: raise ConfigurationError( f"--recover specified but serialization_dir ({serialization_dir}) " "does not exist. There is nothing to recover from.") os.makedirs(serialization_dir, exist_ok=True)
def ensure_model_can_train_save_and_load(self, param_file: str, tolerance: float = 1e-4, cuda_device: int = -1): save_dir = self.TEST_DIR / "save_and_load_test" archive_file = save_dir / "model.tar.gz" model = train_model_from_file(param_file, save_dir) loaded_model = load_archive(archive_file, cuda_device=cuda_device).model state_keys = model.state_dict().keys() loaded_state_keys = loaded_model.state_dict().keys() assert state_keys == loaded_state_keys # First we make sure that the state dict (the parameters) are the same for both models. for key in state_keys: assert_allclose(model.state_dict()[key].cpu().numpy(), loaded_model.state_dict()[key].cpu().numpy(), err_msg=key) params = Params.from_file(param_file) reader = DatasetReader.from_params(params['dataset_reader']) # Need to duplicate params because Iterator.from_params will consume. iterator_params = params['iterator'] iterator_params2 = Params(copy.deepcopy(iterator_params.as_dict())) iterator = DataIterator.from_params(iterator_params) iterator2 = DataIterator.from_params(iterator_params2) # We'll check that even if we index the dataset with each model separately, we still get # the same result out. model_dataset = reader.read(params['validation_data_path']) iterator.index_with(model.vocab) model_batch = next( iterator(model_dataset, shuffle=False, cuda_device=cuda_device)) loaded_dataset = reader.read(params['validation_data_path']) iterator2.index_with(loaded_model.vocab) loaded_batch = next( iterator2(loaded_dataset, shuffle=False, cuda_device=cuda_device)) # Check gradients are None for non-trainable parameters and check that # trainable parameters receive some gradient if they are trainable. self.check_model_computes_gradients_correctly(model, model_batch) # The datasets themselves should be identical. assert model_batch.keys() == loaded_batch.keys() for key in model_batch.keys(): self.assert_fields_equal(model_batch[key], loaded_batch[key], key, 1e-6) # Set eval mode, to turn off things like dropout, then get predictions. model.eval() loaded_model.eval() # Models with stateful RNNs need their states reset to have consistent # behavior after loading. for model_ in [model, loaded_model]: for module in model_.modules(): if hasattr(module, 'stateful') and module.stateful: module.reset_states() model_predictions = model(**model_batch) loaded_model_predictions = loaded_model(**loaded_batch) # Check loaded model's loss exists and we can compute gradients, for continuing training. loaded_model_loss = loaded_model_predictions["loss"] assert loaded_model_loss is not None loaded_model_loss.backward() # Both outputs should have the same keys and the values for these keys should be close. for key in model_predictions.keys(): self.assert_fields_equal(model_predictions[key], loaded_model_predictions[key], name=key, tolerance=tolerance) return model, loaded_model