def pretrain_tokenization(self): paths = [str(x) for x in Path("handler/datadir/").glob("*-train.txt")] print(paths) tokenizer = ByteLevelBPETokenizer() tokenizer.train(files=paths, vocab_size=52_000, min_frequency=2, special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"]) tokenizer.save(".", "danbert-small")
paths = [str(x) for x in Path(args.data_dir).glob("**/*.txt")] print("data files") print(paths) # Initialize a tokenizer tokenizer = ByteLevelBPETokenizer() # Customize training tokenizer.train(files=paths, vocab_size=52_000, min_frequency=2, special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"]) # Need to save it to model dir for inference tokenizer.save(args.model_dir) tokenizer = ByteLevelBPETokenizer(os.path.join(args.model_dir, "vocab.json"), os.path.join(args.model_dir, "merges.txt")) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>"))) tokenizer.enable_truncation(max_length=args.token_max_len) print(tokenizer.encode("Nay, but speak not.")) print(tokenizer.encode("Nay, but speak not.").tokens) from transformers import RobertaConfig config = RobertaConfig(vocab_size=args.vocab_size,