def test_with_single_namespace_and_single_model(self, model_name):
dataset = self.single_namespace_dataset
namespace = "namespace_1"
expected_vocab_size = self._get_expected_vocab_size(
dataset, namespace, model_name)
vocab = Vocabulary.from_pretrained_transformer_and_instances(
dataset, {namespace: model_name})
assert vocab.get_vocab_size(namespace) == expected_vocab_size
def test_only_updates_single_namespace_when_multiple_present(
self, model_name):
dataset = self.multiple_namespace_dataset
namespace1 = "namespace_1"
namespace2 = "namespace_2"
namespace1_vocab_size = self._get_expected_vocab_size(
dataset, namespace1, model_name)
namespace2_vocab_size = Vocabulary.from_instances(
dataset).get_vocab_size("namespace_2")
vocab = Vocabulary.from_pretrained_transformer_and_instances(
dataset, {namespace1: model_name})
# Make sure only the desired namespace is extended
assert vocab.get_vocab_size(namespace1) == namespace1_vocab_size
assert vocab.get_vocab_size(namespace2) == namespace2_vocab_size
def test_with_different_models_per_namespace(self, namespace1_model_name,
namespace2_model_name):
dataset = self.multiple_namespace_dataset
namespace1 = "namespace_1"
namespace2 = "namespace_2"
namespace1_vocab_size = self._get_expected_vocab_size(
dataset, namespace1, namespace1_model_name)
namespace2_vocab_size = self._get_expected_vocab_size(
dataset, namespace2, namespace2_model_name)
vocab = Vocabulary.from_pretrained_transformer_and_instances(
dataset, {
namespace1: namespace1_model_name,
namespace2: namespace2_model_name
})
assert vocab.get_vocab_size(namespace1) == namespace1_vocab_size
assert vocab.get_vocab_size(namespace2) == namespace2_vocab_size
