def pretrain_tokenization(self):
paths = [str(x) for x in Path("handler/datadir/").glob("*-train.txt")]
print(paths)
tokenizer = ByteLevelBPETokenizer()
tokenizer.train(files=paths, vocab_size=52_000, min_frequency=2, special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"])
tokenizer.save(".", "danbert-small")
paths = [str(x) for x in Path(args.data_dir).glob("**/*.txt")]
print("data files")
print(paths)
# Initialize a tokenizer
tokenizer = ByteLevelBPETokenizer()
# Customize training
tokenizer.train(files=paths,
vocab_size=52_000,
min_frequency=2,
special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"])
# Need to save it to model dir for inference
tokenizer.save(args.model_dir)
tokenizer = ByteLevelBPETokenizer(os.path.join(args.model_dir, "vocab.json"),
os.path.join(args.model_dir, "merges.txt"))
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")))
tokenizer.enable_truncation(max_length=args.token_max_len)
print(tokenizer.encode("Nay, but speak not."))
print(tokenizer.encode("Nay, but speak not.").tokens)
from transformers import RobertaConfig
config = RobertaConfig(vocab_size=args.vocab_size,