def test_full_tokenizer(self):
tokenizer = AlbertTokenizer(SAMPLE_VOCAB, keep_accents=True)
tokens = tokenizer.tokenize("This is a test")
self.assertListEqual(tokens, ["▁this", "▁is", "▁a", "▁test"])
self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens),
[48, 25, 21, 1289])
tokens = tokenizer.tokenize("I was born in 92000, and this is falsé.")
self.assertListEqual(tokens, [
"▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this",
"▁is", "▁fal", "s", "é", "."
])
ids = tokenizer.convert_tokens_to_ids(tokens)
self.assertListEqual(
ids, [31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9])
back_tokens = tokenizer.convert_ids_to_tokens(ids)
self.assertListEqual(
back_tokens,
[
"▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and",
"▁this", "▁is", "▁fal", "s", "<unk>", "."
],
)
def add_transformers_vocab(vocab, tokenizer_name):
"""Add vocabulary from tokenizers in transformers for use with pre-tokenized data.
These tokenizers have a convert_tokens_to_ids method, but this doesn't do
anything special, so we can just use the standard indexers.
"""
do_lower_case = "uncased" in tokenizer_name
if tokenizer_name.startswith("bert-"):
tokenizer = BertTokenizer.from_pretrained(tokenizer_name,
do_lower_case=do_lower_case)
elif tokenizer_name.startswith("roberta-"):
tokenizer = RobertaTokenizer.from_pretrained(tokenizer_name)
elif tokenizer_name.startswith("albert"):
tokenizer = AlbertTokenizer(
vocab_file="/work/dcml0714/albert/albert_base/30k-clean.model")
elif tokenizer_name.startswith("xlnet-"):
tokenizer = XLNetTokenizer.from_pretrained(tokenizer_name,
do_lower_case=do_lower_case)
elif tokenizer_name.startswith("openai-gpt"):
tokenizer = OpenAIGPTTokenizer.from_pretrained(tokenizer_name)
elif tokenizer_name.startswith("gpt2"):
tokenizer = GPT2Tokenizer.from_pretrained(tokenizer_name)
elif tokenizer_name.startswith("transfo-xl-"):
tokenizer = TransfoXLTokenizer.from_pretrained(tokenizer_name)
elif tokenizer_name.startswith("xlm-"):
tokenizer = XLMTokenizer.from_pretrained(tokenizer_name)
if (tokenizer_name.startswith("openai-gpt")
or tokenizer_name.startswith("gpt2")
or tokenizer_name.startswith("transo-xl-")):
tokenizer.add_special_tokens({
"bos_token": "<start>",
"sep_token": "<delim>",
"cls_token": "<extract>"
})
# TODO: this is another place can be simplified by "model-before-preprocess" reorganization
# we can pass tokenizer created in model here, see issue <TBD>
vocab_size = len(tokenizer)
# do not use tokenizer.vocab_size, it does not include newly added token
ordered_vocab = tokenizer.convert_ids_to_tokens(range(vocab_size))
log.info("Added transformers vocab (%s): %d tokens", tokenizer_name,
len(ordered_vocab))
for word in ordered_vocab:
vocab.add_token_to_namespace(
word, input_module_tokenizer_name(tokenizer_name))
