def test_full_tokenizer(self): vocab_tokens = [ "<unk>", "[CLS]", "[SEP]", "want", "unwanted", "wa", "un", "running", "," ] with open("/tmp/transfo_xl_tokenizer_test.txt", "w", encoding='utf-8') as vocab_writer: vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) vocab_file = vocab_writer.name tokenizer = TransfoXLTokenizer(vocab_file=vocab_file, lower_case=True) tokenizer.build_vocab() os.remove(vocab_file) tokens = tokenizer.tokenize(u"<unk> UNwanted , running") self.assertListEqual(tokens, ["<unk>", "unwanted", ",", "running"]) self.assertListEqual( tokenizer.convert_tokens_to_ids(tokens), [0, 4, 8, 7]) vocab_file = tokenizer.save_vocabulary(vocab_path="/tmp/") tokenizer.from_pretrained(vocab_file) os.remove(vocab_file) tokens = tokenizer.tokenize(u"<unk> UNwanted , running") self.assertListEqual(tokens, ["<unk>", "unwanted", ",", "running"]) self.assertListEqual( tokenizer.convert_tokens_to_ids(tokens), [0, 4, 8, 7])
def test_tokenizer_from_pretrained(self): cache_dir = "/tmp/pytorch_pretrained_bert_test/" for model_name in list(PRETRAINED_VOCAB_ARCHIVE_MAP.keys())[:1]: tokenizer = TransfoXLTokenizer.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) self.assertIsNotNone(tokenizer)
def transformerXLTokenizer(*args, **kwargs): """ Instantiate a Transformer-XL tokenizer adapted from Vocab class in https://github.com/kimiyoung/transformer-xl Args: pretrained_model_name_or_path: Path to pretrained model archive or one of pre-trained vocab configs below. * transfo-xl-wt103 Example: >>> import torch >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'transformerXLTokenizer', 'transfo-xl-wt103') >>> text = "Who was Jim Henson ?" >>> tokenized_text = tokenizer.tokenize(tokenized_text) >>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) """ tokenizer = TransfoXLTokenizer.from_pretrained(*args, **kwargs) return tokenizer