def load(cls, directory: str, epoch: Optional[int] = None) -> "BaseNLPModel":
with open(os.path.join(directory, "meta.json"), encoding="UTF-8") as f:
meta = json.loads(f.read())
with open(os.path.join(directory, "vocab.json")) as f:
vocab = Vocab.from_json(f.read())
module = cls.from_config({"vocab": vocab, **meta})
module._model = tf.keras.models.load_model(
os.path.join(directory, cls._get_model_filename(epoch=epoch))
)
module._built = True
return module
def test_vocab_serialization(tokens, frequencies):
vocab = Vocab(
tokens,
frequencies,
min_frequency=5,
unk_token="<unk>",
pad_token="<pad>",
bos_token="<bos>",
eos_token="<eos>",
)
new_vocab = Vocab.from_json(vocab.to_json())
assert len(vocab) == len(new_vocab)
assert vocab.sorted_tokens == new_vocab.sorted_tokens
assert vocab.pad == new_vocab.pad
assert vocab.unk == new_vocab.unk
assert vocab.bos == new_vocab.bos
assert vocab.eos == new_vocab.eos
assert str(vocab) == str(new_vocab)