"<unk>",
"<mask>",
])
# Save files to disk
tokenizer.save_model("BR_BERTo")
# Test
tokenizer = ByteLevelBPETokenizer(
"./BR_BERTo/vocab.json",
"./BR_BERTo/merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
print(tokenizer.encode("gostei muito dessa ideia".lower()).tokens)
# Model type
# --------------------------------------------------
config = RobertaConfig(
vocab_size=vocab_size,
max_position_embeddings=514,
num_attention_heads=12,
num_hidden_layers=8,
type_vocab_size=1,
)
model = RobertaForMaskedLM(config=config)
print("Params: ", model.num_parameters())
tokenizer = RobertaTokenizerFast.from_pretrained("./BR_BERTo", max_len=512)
# Dataset load
vocab_size=52_000,
min_frequency=2,
special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"])
# Need to save it to model dir for inference
tokenizer.save(args.model_dir)
tokenizer = ByteLevelBPETokenizer(os.path.join(args.model_dir, "vocab.json"),
os.path.join(args.model_dir, "merges.txt"))
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")))
tokenizer.enable_truncation(max_length=args.token_max_len)
print(tokenizer.encode("Nay, but speak not."))
print(tokenizer.encode("Nay, but speak not.").tokens)
from transformers import RobertaConfig
config = RobertaConfig(vocab_size=args.vocab_size,
max_position_embeddings=args.max_position_embeddings,
num_attention_heads=args.num_attention_heads,
num_hidden_layers=args.num_hidden_layers,
type_vocab_size=args.type_vocab_size)
from transformers import RobertaTokenizerFast
tokenizer = RobertaTokenizerFast.from_pretrained(args.model_dir,
max_len=args.token_max_len)
# -*- coding:utf-8 -*-
import os
from argparse import ArgumentParser
from tokenizers.implementations import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing, RobertaProcessing
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("--token_path",
type=str,
nargs='?',
required=True,
help="")
args = parser.parse_args()
inputpath = args.token_path
tokenizer = ByteLevelBPETokenizer(os.path.join(inputpath, "vocab.json"),
os.path.join(inputpath, "merges.txt"),
add_prefix_space=True,
trim_offsets=True,
lowercase=True,
unicode_normalizer="nfkc")
tokenizer._tokenizer.post_processor = RobertaProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
trim_offsets=True,
add_prefix_space=True)
tokenizer.enable_truncation(max_length=512)
tokens = tokenizer.encode("I am Julien\nI am from China.").tokens
print([x.encode('utf-8') for x in tokens])
from tokenizers.implementations import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing
path = "./model"
tokenizer = ByteLevelBPETokenizer(
"{path}/vocab.json".format(path=path),
"{path}/merges.txt".format(path=path),
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
tokenizer.encode("Dette er første testen.")
tokens = tokenizer.encode("Dette er første testen.").tokens
print(tokens)
from tokenizers.implementations import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing
import os.path
import sys
ROOT_DIRECTORY = os.path.join(os.path.dirname(__file__), '..')
sys.path.append(ROOT_DIRECTORY)
tokenizer = ByteLevelBPETokenizer(
os.path.join(ROOT_DIRECTORY, "models/en_cycl_tokenizer/vocab.json"),
os.path.join(ROOT_DIRECTORY, "models/en_cycl_tokenizer/merges.txt"),
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
token_to_encode = "(() ((#$isa #$McCoyTyner-Musician #$Individual)))"
encoded_token = tokenizer.encode(token_to_encode)
print(encoded_token)
print(encoded_token.tokens)
token_to_encode = "Pair of scissors is marketed as office product."
encoded_token = tokenizer.encode(token_to_encode)
print(encoded_token)
print(encoded_token.tokens)
from tokenizers.implementations import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing
tokenizer = ByteLevelBPETokenizer(
"./EsperBERTo/vocab.json",
"./EsperBERTo/merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
tokenizer.encode("Mi estas Julien.")
tokenizer.encode("Mi estas Julien.").tokens
"""## 3. Train a language model from scratch
**Update:** This section follows along the [`run_language_modeling.py`](https://github.com/huggingface/transformers/blob/master/examples/legacy/run_language_modeling.py) script, using our new [`Trainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer.py) directly. Feel free to pick the approach you like best.
> We’ll train a RoBERTa-like model, which is a BERT-like with a couple of changes (check the [documentation](https://huggingface.co/transformers/model_doc/roberta.html) for more details).
As the model is BERT-like, we’ll train it on a task of *Masked language modeling*, i.e. the predict how to fill arbitrary tokens that we randomly mask in the dataset. This is taken care of by the example script.
"""
# Check that we have a GPU
!nvidia-smi
from tokenizers.implementations import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing
tokenizer = ByteLevelBPETokenizer(
"./bert-tokenizer/vocab.json",
"./bert-tokenizer/merges.txt",
)
from tokenizers.processors import BertProcessing
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
### Testing
tokenizer.encode('TOBB ETU NLP&IR team')
PATH = os.getcwd()
SAVE_MODEL = os.getcwd()
tokenizer = ByteLevelBPETokenizer()
tokenizer.train(files="kant.txt",
vocab_size=52_000,
min_frequency=2,
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
tokenizer.save_model(SAVE_MODEL)
tokenizer = ByteLevelBPETokenizer(
SAVE_MODEL + "/vocab.json",
SAVE_MODEL + "/merges.txt",
)
tokenizer.enable_truncation(max_length=512)
print(tokenizer.encode("For it is in reality vain to profess"))
config = BertConfig(
vocab_size=52_000,
max_position_embeddings=514,
num_attention_heads=12,
num_hidden_layers=6,
type_vocab_size=1,
)
tokenizer = BertTokenizer.from_pretrained(SAVE_MODEL, max_len=512)
model = BertForMaskedLM(config=config)
print(model.num_parameters())
dataset = LineByLineTextDataset(