def batch_to_ids(batch: List[List[str]]) -> torch.Tensor:
"""
Converts a batch of tokenized sentences to a tensor representing the sentences with encoded characters
(len(batch), max sentence length, max word length).
Parameters
----------
batch : ``List[List[str]]``, required
A list of tokenized sentences.
Returns
-------
A tensor of padded character ids.
"""
instances = []
indexer = ELMoTokenCharactersIndexer()
for sentence in batch:
tokens = [Token(token) for token in sentence]
field = TextField(tokens, {'character_ids': indexer})
instance = Instance({"elmo": field})
instances.append(instance)
dataset = Batch(instances)
vocab = Vocabulary()
dataset.index_instances(vocab)
return dataset.as_tensor_dict()['elmo']['character_ids']
def test_forward_pass_runs_correctly(self):
"""
Check to make sure a forward pass on an ensemble of two identical copies of a model yields the same
results as the model itself.
"""
bidaf_ensemble = BidafEnsemble([self.model, self.model])
batch = Batch(self.instances)
batch.index_instances(self.vocab)
training_tensors = batch.as_tensor_dict()
bidaf_output_dict = self.model(**training_tensors)
ensemble_output_dict = bidaf_ensemble(**training_tensors)
metrics = self.model.get_metrics(reset=True)
# We've set up the data such that there's a fake answer that consists of the whole
# paragraph. _Any_ valid prediction for that question should produce an F1 of greater than
# zero, while if we somehow haven't been able to load the evaluation data, or there was an
# error with using the evaluation script, this will fail. This makes sure that we've
# loaded the evaluation data correctly and have hooked things up to the official evaluation
# script.
assert metrics['f1'] > 0
assert torch.equal(ensemble_output_dict['best_span'],
bidaf_output_dict['best_span'])
assert ensemble_output_dict['best_span_str'] == bidaf_output_dict[
'best_span_str']
def forward_on_instances(
self, instances: List[Instance]) -> List[Dict[str, numpy.ndarray]]:
"""
Takes a list of :class:`~allennlp.data.instance.Instance`s, converts that text into
arrays using this model's :class:`Vocabulary`, passes those arrays through
:func:`self.forward()` and :func:`self.decode()` (which by default does nothing)
and returns the result. Before returning the result, we convert any
``torch.Tensors`` into numpy arrays and separate the
batched output into a list of individual dicts per instance. Note that typically
this will be faster on a GPU (and conditionally, on a CPU) than repeated calls to
:func:`forward_on_instance`.
Parameters
----------
instances : List[Instance], required
The instances to run the model on.
cuda_device : int, required
The GPU device to use. -1 means use the CPU.
Returns
-------
A list of the models output for each instance.
"""
batch_size = len(instances)
with torch.no_grad():
cuda_device = self._get_prediction_device()
dataset = Batch(instances)
dataset.index_instances(self.vocab)
model_input = dataset.as_tensor_dict(cuda_device=cuda_device)
outputs = self.decode(self(**model_input))
instance_separated_output: List[Dict[str, numpy.ndarray]] = [
{} for _ in dataset.instances
]
for name, output in list(outputs.items()):
if isinstance(output, torch.Tensor):
# NOTE(markn): This is a hack because 0-dim pytorch tensors are not iterable.
# This occurs with batch size 1, because we still want to include the loss in that case.
if output.dim() == 0:
output = output.unsqueeze(0)
if output.size(0) != batch_size:
self._maybe_warn_for_unseparable_batches(name)
continue
output = output.detach().cpu().numpy()
elif len(output) != batch_size:
self._maybe_warn_for_unseparable_batches(name)
continue
outputs[name] = output
for instance_output, batch_element in zip(
instance_separated_output, output):
instance_output[name] = batch_element
return instance_separated_output
def ensure_batch_predictions_are_consistent(self):
self.model.eval()
single_predictions = []
for i, instance in enumerate(self.instances):
dataset = Batch([instance])
tensors = dataset.as_tensor_dict(dataset.get_padding_lengths())
result = self.model(**tensors)
single_predictions.append(result)
full_dataset = Batch(self.instances)
batch_tensors = full_dataset.as_tensor_dict(
full_dataset.get_padding_lengths())
batch_predictions = self.model(**batch_tensors)
for i, instance_predictions in enumerate(single_predictions):
for key, single_predicted in instance_predictions.items():
tolerance = 1e-6
if 'loss' in key:
# Loss is particularly unstable; we'll just be satisfied if everything else is
# close.
continue
single_predicted = single_predicted[0]
batch_predicted = batch_predictions[key][i]
if isinstance(single_predicted, torch.Tensor):
if single_predicted.size() != batch_predicted.size():
slices = tuple(
slice(0, size) for size in single_predicted.size())
batch_predicted = batch_predicted[slices]
assert_allclose(single_predicted.data.numpy(),
batch_predicted.data.numpy(),
atol=tolerance,
err_msg=key)
else:
assert single_predicted == batch_predicted, key
def test_tagger_with_elmo_token_embedder_forward_pass_runs_correctly(self):
dataset = Batch(self.instances)
dataset.index_instances(self.vocab)
training_tensors = dataset.as_tensor_dict()
output_dict = self.model(**training_tensors)
tags = output_dict['tags']
assert len(tags) == 2
assert len(tags[0]) == 7
assert len(tags[1]) == 7
for example_tags in tags:
for tag_id in example_tags:
tag = self.model.vocab.get_token_from_index(tag_id,
namespace="labels")
assert tag in {'O', 'I-ORG', 'I-PER', 'I-LOC'}
def _sentences_to_ids(self, sentences):
indexer = ELMoTokenCharactersIndexer()
# For each sentence, first create a TextField, then create an instance
instances = []
for sentence in sentences:
tokens = [Token(token) for token in sentence]
field = TextField(tokens, {'character_ids': indexer})
instance = Instance({'elmo': field})
instances.append(instance)
dataset = Batch(instances)
vocab = Vocabulary()
dataset.index_instances(vocab)
return dataset.as_tensor_dict()['elmo']['character_ids']
def setUp(self):
token_indexer = SingleIdTokenIndexer("tokens")
text_field = TextField([Token(t) for t in ["a", "a", "a", "a", "b", "b", "c", "c", "c"]],
{"tokens": token_indexer})
self.instance = Instance({"text": text_field})
self.dataset = Batch([self.instance])
super(TestVocabulary, self).setUp()
def _create_batches(self, instances: Iterable[Instance],
shuffle: bool) -> Iterable[Batch]:
for instance_list in self._memory_sized_lists(instances):
instance_list = sort_by_padding(instance_list, self._sorting_keys,
self.vocab, self._padding_noise)
batches = []
for batch_instances in lazy_groups_of(iter(instance_list),
self._batch_size):
for possibly_smaller_batches in self._ensure_batch_is_sufficiently_small(
batch_instances):
batches.append(Batch(possibly_smaller_batches))
move_to_front = self._biggest_batch_first and len(batches) > 1
if move_to_front:
# We'll actually pop the last _two_ batches, because the last one might not be full.
last_batch = batches.pop()
penultimate_batch = batches.pop()
if shuffle:
random.shuffle(batches)
else:
logger.warning(
"shuffle parameter is set to False,"
" while bucket iterators by definition change the order of your data."
)
if move_to_front:
batches.insert(0, penultimate_batch)
batches.insert(0, last_batch)
yield from batches
def get_vocab_and_both_elmo_indexed_ids(batch: List[List[str]]):
instances = []
indexer = ELMoTokenCharactersIndexer()
indexer2 = SingleIdTokenIndexer()
for sentence in batch:
tokens = [Token(token) for token in sentence]
field = TextField(tokens, {
'character_ids': indexer,
'tokens': indexer2
})
instance = Instance({"elmo": field})
instances.append(instance)
dataset = Batch(instances)
vocab = Vocabulary.from_instances(instances)
dataset.index_instances(vocab)
return vocab, dataset.as_tensor_dict()["elmo"]
def _create_batches(self, instances: Iterable[Instance], shuffle: bool) -> Iterable[Batch]:
# First break the dataset into memory-sized lists:
for instance_list in self._memory_sized_lists(instances):
if shuffle:
random.shuffle(instance_list)
iterator = iter(instance_list)
# Then break each memory-sized list into batches.
for batch_instances in lazy_groups_of(iterator, self._batch_size):
for possibly_smaller_batches in self._ensure_batch_is_sufficiently_small(batch_instances):
batch = Batch(possibly_smaller_batches)
yield batch
def test_invalid_vocab_extension(self):
vocab_dir = self.TEST_DIR / 'vocab_save'
original_vocab = Vocabulary(non_padded_namespaces=["tokens1"])
original_vocab.add_token_to_namespace("a", namespace="tokens1")
original_vocab.add_token_to_namespace("b", namespace="tokens1")
original_vocab.add_token_to_namespace("p", namespace="tokens2")
original_vocab.save_to_files(vocab_dir)
text_field1 = TextField([Token(t) for t in ["a" "c"]],
{"tokens1": SingleIdTokenIndexer("tokens1")})
text_field2 = TextField([Token(t) for t in ["p", "q", "r"]],
{"tokens2": SingleIdTokenIndexer("tokens2")})
instances = Batch([Instance({"text1": text_field1, "text2": text_field2})])
# Following 2 should give error: token1 is non-padded in original_vocab but not in instances
params = Params({"directory_path": vocab_dir, "extend": True,
"non_padded_namespaces": []})
with pytest.raises(ConfigurationError):
_ = Vocabulary.from_params(params, instances)
with pytest.raises(ConfigurationError):
extended_vocab = copy.copy(original_vocab)
params = Params({"non_padded_namespaces": []})
extended_vocab.extend_from_instances(params, instances)
with pytest.raises(ConfigurationError):
extended_vocab = copy.copy(original_vocab)
extended_vocab._extend(non_padded_namespaces=[],
tokens_to_add={"tokens1": ["a"], "tokens2": ["p"]})
# Following 2 should not give error: overlapping namespaces have same padding setting
params = Params({"directory_path": vocab_dir, "extend": True,
"non_padded_namespaces": ["tokens1"]})
Vocabulary.from_params(params, instances)
extended_vocab = copy.copy(original_vocab)
params = Params({"non_padded_namespaces": ["tokens1"]})
extended_vocab.extend_from_instances(params, instances)
extended_vocab = copy.copy(original_vocab)
extended_vocab._extend(non_padded_namespaces=["tokens1"],
tokens_to_add={"tokens1": ["a"], "tokens2": ["p"]})
# Following 2 should give error: token1 is padded in instances but not in original_vocab
params = Params({"directory_path": vocab_dir, "extend": True,
"non_padded_namespaces": ["tokens1", "tokens2"]})
with pytest.raises(ConfigurationError):
_ = Vocabulary.from_params(params, instances)
with pytest.raises(ConfigurationError):
extended_vocab = copy.copy(original_vocab)
params = Params({"non_padded_namespaces": ["tokens1", "tokens2"]})
extended_vocab.extend_from_instances(params, instances)
with pytest.raises(ConfigurationError):
extended_vocab = copy.copy(original_vocab)
extended_vocab._extend(non_padded_namespaces=["tokens1", "tokens2"],
tokens_to_add={"tokens1": ["a"], "tokens2": ["p"]})
def test_max_vocab_size_partial_dict(self):
indexers = {"tokens": SingleIdTokenIndexer(), "token_characters": TokenCharactersIndexer()}
instance = Instance({
'text': TextField([Token(w) for w in 'Abc def ghi jkl mno pqr stu vwx yz'.split(' ')], indexers)
})
dataset = Batch([instance])
params = Params({
"max_vocab_size": {
"tokens": 1
}
})
vocab = Vocabulary.from_params(params=params, instances=dataset)
assert len(vocab.get_index_to_token_vocabulary("tokens").values()) == 3 # 1 + 2
assert len(vocab.get_index_to_token_vocabulary("token_characters").values()) == 28 # 26 + 2
def set_up_model(self, param_file, dataset_file):
# pylint: disable=attribute-defined-outside-init
self.param_file = param_file
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
instances = reader.read(dataset_file)
# Use parameters for vocabulary if they are present in the config file, so that choices like
# "non_padded_namespaces", "min_count" etc. can be set if needed.
if 'vocabulary' in params:
vocab_params = params['vocabulary']
vocab = Vocabulary.from_params(params=vocab_params,
instances=instances)
else:
vocab = Vocabulary.from_instances(instances)
self.vocab = vocab
self.instances = instances
self.model = Model.from_params(vocab=self.vocab,
params=params['model'])
# TODO(joelgrus) get rid of these
# (a lot of the model tests use them, so they'll have to be changed)
self.dataset = Batch(self.instances)
self.dataset.index_instances(self.vocab)
def test_forward_pass_runs_correctly(self):
batch = Batch(self.instances)
batch.index_instances(self.vocab)
training_tensors = batch.as_tensor_dict()
output_dict = self.model(**training_tensors)
metrics = self.model.get_metrics(reset=True)
# We've set up the data such that there's a fake answer that consists of the whole
# paragraph. _Any_ valid prediction for that question should produce an F1 of greater than
# zero, while if we somehow haven't been able to load the evaluation data, or there was an
# error with using the evaluation script, this will fail. This makes sure that we've
# loaded the evaluation data correctly and have hooked things up to the official evaluation
# script.
assert metrics['f1'] > 0
span_start_probs = output_dict['span_start_probs'][0].data.numpy()
span_end_probs = output_dict['span_start_probs'][0].data.numpy()
assert_almost_equal(numpy.sum(span_start_probs, -1), 1, decimal=6)
assert_almost_equal(numpy.sum(span_end_probs, -1), 1, decimal=6)
span_start, span_end = tuple(output_dict['best_span'][0].data.numpy())
assert span_start >= 0
assert span_start <= span_end
assert span_end < self.instances[0].fields['passage'].sequence_length()
assert isinstance(output_dict['best_span_str'][0], str)
def test_as_tensor_dict(self):
dataset = Batch(self.instances)
dataset.index_instances(self.vocab)
padding_lengths = dataset.get_padding_lengths()
tensors = dataset.as_tensor_dict(padding_lengths)
text1 = tensors["text1"]["tokens"].detach().cpu().numpy()
text2 = tensors["text2"]["tokens"].detach().cpu().numpy()
numpy.testing.assert_array_almost_equal(text1, numpy.array([[2, 3, 4, 5, 6],
[1, 3, 4, 5, 6]]))
numpy.testing.assert_array_almost_equal(text2, numpy.array([[2, 3, 4, 1, 5, 6],
[2, 3, 1, 0, 0, 0]]))
def dry_run_from_params(params: Params, serialization_dir: str) -> None:
prepare_environment(params)
vocab_params = params.pop("vocabulary", {})
os.makedirs(serialization_dir, exist_ok=True)
vocab_dir = os.path.join(serialization_dir, "vocabulary")
if os.path.isdir(vocab_dir) and os.listdir(vocab_dir) is not None:
raise ConfigurationError("The 'vocabulary' directory in the provided "
"serialization directory is non-empty")
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info("From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation))
instances = [instance for key, dataset in all_datasets.items()
for instance in dataset
if key in datasets_for_vocab_creation]
vocab = Vocabulary.from_params(vocab_params, instances)
dataset = Batch(instances)
dataset.index_instances(vocab)
dataset.print_statistics()
vocab.print_statistics()
logger.info(f"writing the vocabulary to {vocab_dir}.")
vocab.save_to_files(vocab_dir)
model = Model.from_params(vocab=vocab, params=params.pop('model'))
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
frozen_parameter_names, tunable_parameter_names = \
get_frozen_and_tunable_parameter_names(model)
logger.info("Following parameters are Frozen (without gradient):")
for name in frozen_parameter_names:
logger.info(name)
logger.info("Following parameters are Tunable (with gradient):")
for name in tunable_parameter_names:
logger.info(name)
def test_saving_and_loading_works_with_byte_encoding(self):
# We're going to set a vocabulary from a TextField using byte encoding, index it, save the
# vocab, load the vocab, then index the text field again, and make sure we get the same
# result.
tokenizer = CharacterTokenizer(byte_encoding='utf-8')
token_indexer = TokenCharactersIndexer(character_tokenizer=tokenizer)
tokens = [Token(t) for t in ["Øyvind", "für", "汉字"]]
text_field = TextField(tokens, {"characters": token_indexer})
dataset = Batch([Instance({"sentence": text_field})])
vocab = Vocabulary.from_instances(dataset)
text_field.index(vocab)
indexed_tokens = deepcopy(text_field._indexed_tokens) # pylint: disable=protected-access
vocab_dir = self.TEST_DIR / 'vocab_save'
vocab.save_to_files(vocab_dir)
vocab2 = Vocabulary.from_files(vocab_dir)
text_field2 = TextField(tokens, {"characters": token_indexer})
text_field2.index(vocab2)
indexed_tokens2 = deepcopy(text_field2._indexed_tokens) # pylint: disable=protected-access
assert indexed_tokens == indexed_tokens2
def test_from_params_extend_config(self):
vocab_dir = self.TEST_DIR / 'vocab_save'
original_vocab = Vocabulary(non_padded_namespaces=["tokens"])
original_vocab.add_token_to_namespace("a", namespace="tokens")
original_vocab.save_to_files(vocab_dir)
text_field = TextField([Token(t) for t in ["a", "b"]],
{"tokens": SingleIdTokenIndexer("tokens")})
instances = Batch([Instance({"text": text_field})])
# If you ask to extend vocab from `directory_path`, instances must be passed
# in Vocabulary constructor, or else there is nothing to extend to.
params = Params({"directory_path": vocab_dir, "extend": True})
with pytest.raises(ConfigurationError):
_ = Vocabulary.from_params(params)
# If you ask to extend vocab, `directory_path` key must be present in params,
# or else there is nothing to extend from.
params = Params({"extend": True})
with pytest.raises(ConfigurationError):
_ = Vocabulary.from_params(params, instances)
class ModelTestCase(AllenNlpTestCase):
"""
A subclass of :class:`~allennlp.common.testing.test_case.AllenNlpTestCase`
with added methods for testing :class:`~allennlp.models.model.Model` subclasses.
"""
def set_up_model(self, param_file, dataset_file):
# pylint: disable=attribute-defined-outside-init
self.param_file = param_file
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
instances = reader.read(dataset_file)
# Use parameters for vocabulary if they are present in the config file, so that choices like
# "non_padded_namespaces", "min_count" etc. can be set if needed.
if 'vocabulary' in params:
vocab_params = params['vocabulary']
vocab = Vocabulary.from_params(params=vocab_params,
instances=instances)
else:
vocab = Vocabulary.from_instances(instances)
self.vocab = vocab
self.instances = instances
self.model = Model.from_params(vocab=self.vocab,
params=params['model'])
# TODO(joelgrus) get rid of these
# (a lot of the model tests use them, so they'll have to be changed)
self.dataset = Batch(self.instances)
self.dataset.index_instances(self.vocab)
def ensure_model_can_train_save_and_load(self,
param_file: str,
tolerance: float = 1e-4,
cuda_device: int = -1):
save_dir = self.TEST_DIR / "save_and_load_test"
archive_file = save_dir / "model.tar.gz"
model = train_model_from_file(param_file, save_dir)
loaded_model = load_archive(archive_file,
cuda_device=cuda_device).model
state_keys = model.state_dict().keys()
loaded_state_keys = loaded_model.state_dict().keys()
assert state_keys == loaded_state_keys
# First we make sure that the state dict (the parameters) are the same for both models.
for key in state_keys:
assert_allclose(model.state_dict()[key].cpu().numpy(),
loaded_model.state_dict()[key].cpu().numpy(),
err_msg=key)
params = Params.from_file(param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
# Need to duplicate params because Iterator.from_params will consume.
iterator_params = params['iterator']
iterator_params2 = Params(copy.deepcopy(iterator_params.as_dict()))
iterator = DataIterator.from_params(iterator_params)
iterator2 = DataIterator.from_params(iterator_params2)
# We'll check that even if we index the dataset with each model separately, we still get
# the same result out.
model_dataset = reader.read(params['validation_data_path'])
iterator.index_with(model.vocab)
model_batch = next(
iterator(model_dataset, shuffle=False, cuda_device=cuda_device))
loaded_dataset = reader.read(params['validation_data_path'])
iterator2.index_with(loaded_model.vocab)
loaded_batch = next(
iterator2(loaded_dataset, shuffle=False, cuda_device=cuda_device))
# Check gradients are None for non-trainable parameters and check that
# trainable parameters receive some gradient if they are trainable.
self.check_model_computes_gradients_correctly(model, model_batch)
# The datasets themselves should be identical.
assert model_batch.keys() == loaded_batch.keys()
for key in model_batch.keys():
self.assert_fields_equal(model_batch[key], loaded_batch[key], key,
1e-6)
# Set eval mode, to turn off things like dropout, then get predictions.
model.eval()
loaded_model.eval()
# Models with stateful RNNs need their states reset to have consistent
# behavior after loading.
for model_ in [model, loaded_model]:
for module in model_.modules():
if hasattr(module, 'stateful') and module.stateful:
module.reset_states()
model_predictions = model(**model_batch)
loaded_model_predictions = loaded_model(**loaded_batch)
# Check loaded model's loss exists and we can compute gradients, for continuing training.
loaded_model_loss = loaded_model_predictions["loss"]
assert loaded_model_loss is not None
loaded_model_loss.backward()
# Both outputs should have the same keys and the values for these keys should be close.
for key in model_predictions.keys():
self.assert_fields_equal(model_predictions[key],
loaded_model_predictions[key],
name=key,
tolerance=tolerance)
return model, loaded_model
def assert_fields_equal(self,
field1,
field2,
name: str,
tolerance: float = 1e-6) -> None:
if isinstance(field1, torch.Tensor):
assert_allclose(field1.detach().cpu().numpy(),
field2.detach().cpu().numpy(),
rtol=tolerance,
err_msg=name)
elif isinstance(field1, dict):
assert field1.keys() == field2.keys()
for key in field1:
self.assert_fields_equal(field1[key],
field2[key],
tolerance=tolerance,
name=name + '.' + str(key))
elif isinstance(field1, (list, tuple)):
assert len(field1) == len(field2)
for i, (subfield1, subfield2) in enumerate(zip(field1, field2)):
self.assert_fields_equal(subfield1,
subfield2,
tolerance=tolerance,
name=name + f"[{i}]")
elif isinstance(field1, (float, int)):
assert_allclose([field1], [field2], rtol=tolerance, err_msg=name)
else:
assert field1 == field2
@staticmethod
def check_model_computes_gradients_correctly(model, model_batch):
model.zero_grad()
result = model(**model_batch)
result["loss"].backward()
has_zero_or_none_grads = {}
for name, parameter in model.named_parameters():
zeros = torch.zeros(parameter.size())
if parameter.requires_grad:
if parameter.grad is None:
has_zero_or_none_grads[
name] = "No gradient computed (i.e parameter.grad is None)"
elif parameter.grad.is_sparse or parameter.grad.data.is_sparse:
pass
# Some parameters will only be partially updated,
# like embeddings, so we just check that any gradient is non-zero.
elif (parameter.grad.cpu() == zeros).all():
has_zero_or_none_grads[
name] = f"zeros with shape ({tuple(parameter.grad.size())})"
else:
assert parameter.grad is None
if has_zero_or_none_grads:
for name, grad in has_zero_or_none_grads.items():
print(f"Parameter: {name} had incorrect gradient: {grad}")
raise Exception(
"Incorrect gradients found. See stdout for more info.")
def ensure_batch_predictions_are_consistent(self):
self.model.eval()
single_predictions = []
for i, instance in enumerate(self.instances):
dataset = Batch([instance])
tensors = dataset.as_tensor_dict(dataset.get_padding_lengths())
result = self.model(**tensors)
single_predictions.append(result)
full_dataset = Batch(self.instances)
batch_tensors = full_dataset.as_tensor_dict(
full_dataset.get_padding_lengths())
batch_predictions = self.model(**batch_tensors)
for i, instance_predictions in enumerate(single_predictions):
for key, single_predicted in instance_predictions.items():
tolerance = 1e-6
if 'loss' in key:
# Loss is particularly unstable; we'll just be satisfied if everything else is
# close.
continue
single_predicted = single_predicted[0]
batch_predicted = batch_predictions[key][i]
if isinstance(single_predicted, torch.Tensor):
if single_predicted.size() != batch_predicted.size():
slices = tuple(
slice(0, size) for size in single_predicted.size())
batch_predicted = batch_predicted[slices]
assert_allclose(single_predicted.data.numpy(),
batch_predicted.data.numpy(),
atol=tolerance,
err_msg=key)
else:
assert single_predicted == batch_predicted, key
def test_padding_lengths_uses_max_instance_lengths(self):
dataset = Batch(self.instances)
dataset.index_instances(self.vocab)
padding_lengths = dataset.get_padding_lengths()
assert padding_lengths == {"text1": {"num_tokens": 5}, "text2": {"num_tokens": 6}}
def test_instances_must_have_homogeneous_fields(self):
instance1 = Instance({"tag": (LabelField(1, skip_indexing=True))})
instance2 = Instance({"words": TextField([Token("hello")], {})})
with pytest.raises(ConfigurationError):
_ = Batch([instance1, instance2])
def test_valid_vocab_extension(self):
vocab_dir = self.TEST_DIR / 'vocab_save'
extension_ways = ["from_params", "extend_from_instances"]
# Test: padded/non-padded common namespaces are extending appropriately
non_padded_namespaces_list = [[], ["tokens"]]
for non_padded_namespaces in non_padded_namespaces_list:
original_vocab = Vocabulary(non_padded_namespaces=non_padded_namespaces)
original_vocab.add_token_to_namespace("d", namespace="tokens")
original_vocab.add_token_to_namespace("a", namespace="tokens")
original_vocab.add_token_to_namespace("b", namespace="tokens")
text_field = TextField([Token(t) for t in ["a", "d", "c", "e"]],
{"tokens": SingleIdTokenIndexer("tokens")})
instances = Batch([Instance({"text": text_field})])
for way in extension_ways:
if way == "extend_from_instances":
extended_vocab = copy.copy(original_vocab)
params = Params({"non_padded_namespaces": non_padded_namespaces})
extended_vocab.extend_from_instances(params, instances)
else:
shutil.rmtree(vocab_dir, ignore_errors=True)
original_vocab.save_to_files(vocab_dir)
params = Params({"directory_path": vocab_dir, "extend": True,
"non_padded_namespaces": non_padded_namespaces})
extended_vocab = Vocabulary.from_params(params, instances)
extra_count = 2 if extended_vocab.is_padded("tokens") else 0
assert extended_vocab.get_token_index("d", "tokens") == 0 + extra_count
assert extended_vocab.get_token_index("a", "tokens") == 1 + extra_count
assert extended_vocab.get_token_index("b", "tokens") == 2 + extra_count
assert extended_vocab.get_token_index("c", "tokens") # should be present
assert extended_vocab.get_token_index("e", "tokens") # should be present
assert extended_vocab.get_vocab_size("tokens") == 5 + extra_count
# Test: padded/non-padded non-common namespaces are extending appropriately
non_padded_namespaces_list = [[],
["tokens1"],
["tokens1", "tokens2"]]
for non_padded_namespaces in non_padded_namespaces_list:
original_vocab = Vocabulary(non_padded_namespaces=non_padded_namespaces)
original_vocab.add_token_to_namespace("a", namespace="tokens1") # index2
text_field = TextField([Token(t) for t in ["b"]],
{"tokens2": SingleIdTokenIndexer("tokens2")})
instances = Batch([Instance({"text": text_field})])
for way in extension_ways:
if way == "extend_from_instances":
extended_vocab = copy.copy(original_vocab)
params = Params({"non_padded_namespaces": non_padded_namespaces})
extended_vocab.extend_from_instances(params, instances)
else:
shutil.rmtree(vocab_dir, ignore_errors=True)
original_vocab.save_to_files(vocab_dir)
params = Params({"directory_path": vocab_dir, "extend": True,
"non_padded_namespaces": non_padded_namespaces})
extended_vocab = Vocabulary.from_params(params, instances)
# Should have two namespaces
assert len(extended_vocab._token_to_index) == 2
extra_count = 2 if extended_vocab.is_padded("tokens1") else 0
assert extended_vocab.get_vocab_size("tokens1") == 1 + extra_count
extra_count = 2 if extended_vocab.is_padded("tokens2") else 0
assert extended_vocab.get_vocab_size("tokens2") == 1 + extra_count
def test_from_params_valid_vocab_extension_thoroughly(self):
'''
Tests for Valid Vocab Extension thoroughly: Vocab extension is valid
when overlapping namespaces have same padding behaviour (padded/non-padded)
Summary of namespace paddings in this test:
original_vocab namespaces
tokens0 padded
tokens1 non-padded
tokens2 padded
tokens3 non-padded
instances namespaces
tokens0 padded
tokens1 non-padded
tokens4 padded
tokens5 non-padded
TypicalExtention example: (of tokens1 namespace)
-> original_vocab index2token
apple #0->apple
bat #1->bat
cat #2->cat
-> Token to be extended with: cat, an, apple, banana, atom, bat
-> extended_vocab: index2token
apple #0->apple
bat #1->bat
cat #2->cat
an #3->an
atom #4->atom
banana #5->banana
'''
vocab_dir = self.TEST_DIR / 'vocab_save'
original_vocab = Vocabulary(non_padded_namespaces=["tokens1", "tokens3"])
original_vocab.add_token_to_namespace("apple", namespace="tokens0") # index:2
original_vocab.add_token_to_namespace("bat", namespace="tokens0") # index:3
original_vocab.add_token_to_namespace("cat", namespace="tokens0") # index:4
original_vocab.add_token_to_namespace("apple", namespace="tokens1") # index:0
original_vocab.add_token_to_namespace("bat", namespace="tokens1") # index:1
original_vocab.add_token_to_namespace("cat", namespace="tokens1") # index:2
original_vocab.add_token_to_namespace("a", namespace="tokens2") # index:0
original_vocab.add_token_to_namespace("b", namespace="tokens2") # index:1
original_vocab.add_token_to_namespace("c", namespace="tokens2") # index:2
original_vocab.add_token_to_namespace("p", namespace="tokens3") # index:0
original_vocab.add_token_to_namespace("q", namespace="tokens3") # index:1
original_vocab.save_to_files(vocab_dir)
text_field0 = TextField([Token(t) for t in ["cat", "an", "apple", "banana", "atom", "bat"]],
{"tokens0": SingleIdTokenIndexer("tokens0")})
text_field1 = TextField([Token(t) for t in ["cat", "an", "apple", "banana", "atom", "bat"]],
{"tokens1": SingleIdTokenIndexer("tokens1")})
text_field4 = TextField([Token(t) for t in ["l", "m", "n", "o"]],
{"tokens4": SingleIdTokenIndexer("tokens4")})
text_field5 = TextField([Token(t) for t in ["x", "y", "z"]],
{"tokens5": SingleIdTokenIndexer("tokens5")})
instances = Batch([Instance({"text0": text_field0, "text1": text_field1,
"text4": text_field4, "text5": text_field5})])
params = Params({"directory_path": vocab_dir,
"extend": True,
"non_padded_namespaces": ["tokens1", "tokens5"]})
extended_vocab = Vocabulary.from_params(params, instances)
# namespaces: tokens0, tokens1 is common.
# tokens2, tokens3 only vocab has. tokens4, tokens5 only instances
extended_namespaces = {*extended_vocab._token_to_index}
assert extended_namespaces == {"tokens{}".format(i) for i in range(6)}
# # Check that _non_padded_namespaces list is consistent after extension
assert extended_vocab._non_padded_namespaces == {"tokens1", "tokens3", "tokens5"}
# # original_vocab["tokens1"] has 3 tokens, instances of "tokens1" ns has 5 tokens. 2 overlapping
assert extended_vocab.get_vocab_size("tokens1") == 6
assert extended_vocab.get_vocab_size("tokens0") == 8 # 2 extra overlapping because padded
# namespace tokens3, tokens4 was only in original_vocab,
# and its token count should be same in extended_vocab
assert extended_vocab.get_vocab_size("tokens2") == original_vocab.get_vocab_size("tokens2")
assert extended_vocab.get_vocab_size("tokens3") == original_vocab.get_vocab_size("tokens3")
# namespace tokens2 was only in instances,
# and its token count should be same in extended_vocab
assert extended_vocab.get_vocab_size("tokens4") == 6 # l,m,n,o + oov + padding
assert extended_vocab.get_vocab_size("tokens5") == 3 # x,y,z
# Word2index mapping of all words in all namespaces of original_vocab
# should be maintained in extended_vocab
for namespace, token2index in original_vocab._token_to_index.items():
for token, _ in token2index.items():
vocab_index = original_vocab.get_token_index(token, namespace)
extended_vocab_index = extended_vocab.get_token_index(token, namespace)
assert vocab_index == extended_vocab_index
# And same for Index2Word mapping
for namespace, index2token in original_vocab._index_to_token.items():
for index, _ in index2token.items():
vocab_token = original_vocab.get_token_from_index(index, namespace)
extended_vocab_token = extended_vocab.get_token_from_index(index, namespace)
assert vocab_token == extended_vocab_token