def test_language_to_from_disk(nlp, name): doc = nlp("hello world") assert is_valid_tensor(doc.tensor) with make_tempdir() as tempdir: nlp.to_disk(tempdir) new_nlp = PyTT_Language() new_nlp.add_pipe(new_nlp.create_pipe("sentencizer")) wordpiecer = PyTT_WordPiecer(new_nlp.vocab, pytt_name=name) tok2vec = PyTT_TokenVectorEncoder(new_nlp.vocab, pytt_name=name) new_nlp.add_pipe(wordpiecer) new_nlp.add_pipe(tok2vec) new_nlp.from_disk(tempdir) assert new_nlp.pipe_names == nlp.pipe_names new_doc = new_nlp("hello world") assert is_valid_tensor(new_doc.tensor) assert_equal(doc.tensor, new_doc.tensor)
def main(path, name="bert-base-uncased", lang="en"): msg = Printer() msg.info(f"Creating model for '{name}' ({lang})") with msg.loading(f"Setting up the pipeline..."): nlp = PyTT_Language(pytt_name=name, meta={"lang": lang}) nlp.add_pipe(nlp.create_pipe("sentencizer")) nlp.add_pipe(PyTT_WordPiecer.from_pretrained(nlp.vocab, name)) nlp.add_pipe(PyTT_TokenVectorEncoder.from_pretrained(nlp.vocab, name)) msg.good("Initialized the model pipeline") nlp.to_disk(path) msg.good(f"Saved '{name}' ({lang})") msg.text(f"Pipeline: {nlp.pipe_names}") msg.text(f"Location: {path}") with msg.loading("Verifying model loads..."): nlp.from_disk(path) msg.good("Model loads!")