def get_recurrent_tokenizer(vocab,
max_context_tokens,
unk_token,
pad_token,
device="cpu"):
"""
Return a tokenizer to be used with recurrent-based models
"""
question_tokenizer = Tokenizer(WordLevel(vocab, unk_token=unk_token))
question_tokenizer.normalizer = Sequence(
[StripAccents(), Lowercase(), Strip()])
question_tokenizer.pre_tokenizer = PreSequence(
[Whitespace(), Punctuation()])
question_tokenizer.enable_padding(direction="right",
pad_id=vocab[pad_token],
pad_type_id=1,
pad_token=pad_token)
context_tokenizer = Tokenizer(WordLevel(vocab, unk_token=unk_token))
context_tokenizer.normalizer = Sequence(
[StripAccents(), Lowercase(), Strip()])
context_tokenizer.pre_tokenizer = PreSequence(
[Whitespace(), Punctuation()])
context_tokenizer.enable_padding(
direction="right",
pad_id=vocab[pad_token],
pad_type_id=1,
pad_token=pad_token,
)
context_tokenizer.enable_truncation(max_context_tokens)
return RecurrentSquadTokenizer(question_tokenizer,
context_tokenizer,
device=device)
def __init__(
self,
vocab: Optional[Union[str, Dict[str, int]]] = None,
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int,
int]]]] = None,
add_prefix_space: bool = False,
lowercase: bool = False,
dropout: Optional[float] = None,
unicode_normalizer: Optional[str] = None,
continuing_subword_prefix: Optional[str] = None,
end_of_word_suffix: Optional[str] = None,
trim_offsets: bool = False,
):
if vocab is not None and merges is not None:
tokenizer = Tokenizer(
BPE(
vocab,
merges,
dropout=dropout,
continuing_subword_prefix=continuing_subword_prefix or "",
end_of_word_suffix=end_of_word_suffix or "",
))
else:
tokenizer = Tokenizer(BPE())
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
if lowercase:
normalizers += [Lowercase()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.post_processor = processors.ByteLevel(
trim_offsets=trim_offsets)
parameters = {
"model": "ByteLevelBPE",
"add_prefix_space": add_prefix_space,
"lowercase": lowercase,
"dropout": dropout,
"unicode_normalizer": unicode_normalizer,
"continuing_subword_prefix": continuing_subword_prefix,
"end_of_word_suffix": end_of_word_suffix,
"trim_offsets": trim_offsets,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
add_prefix_space: bool = False,
lowercase: bool = False,
dropout: Optional[float] = None,
unicode_normalizer: Optional[str] = None,
continuing_subword_prefix: Optional[str] = None,
end_of_word_suffix: Optional[str] = None,
):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE.from_files(
vocab_file,
merges_file,
dropout=dropout,
continuing_subword_prefix=continuing_subword_prefix or "",
end_of_word_suffix=end_of_word_suffix or "",
))
else:
tokenizer = Tokenizer(BPE.empty())
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
if lowercase:
normalizers += [Lowercase()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.ByteLevel()
parameters = {
"model": "ByteLevelBPE",
"add_prefix_space": add_prefix_space,
"lowercase": lowercase,
"dropout": dropout,
"unicode_normalizer": unicode_normalizer,
"continuing_subword_prefix": continuing_subword_prefix,
"end_of_word_suffix": end_of_word_suffix,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
vocab_file: Optional[str] = None,
unk_token: Union[str, AddedToken] = "[UNK]",
pad_token: Union[str, AddedToken] = "[PAD]",
mask_token: Union[str, AddedToken] = "[MASK]",
lowercase: bool = False,
unicode_normalizer: Optional[str] = None,
):
if vocab_file is not None:
logging.info(f"Initiating tokenizer at {vocab_file}")
tokenizer = Tokenizer(
WordLevel(vocab=vocab_file, unk_token=unk_token))
else:
tokenizer = Tokenizer(WordLevel(unk_token=unk_token))
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
if tokenizer.token_to_id(str(pad_token)) is not None:
tokenizer.add_special_tokens([str(pad_token)])
if tokenizer.token_to_id(str(mask_token)) is not None:
tokenizer.add_special_tokens([str(mask_token)])
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
if lowercase:
normalizers += [Lowercase()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
parameters = {
"model": "WordLevel",
"unk_token": unk_token,
"pad_token": pad_token,
"mask_token": mask_token,
"lowercase": lowercase,
"unicode_normalizer": unicode_normalizer,
}
super().__init__(tokenizer, parameters)
def build(self, afm: AuxiliaryFileManager,
corpus: AuxiliaryFile) -> AuxiliaryFile:
subset = self._create_subset_file(afm, corpus)
# Create WordPiece model with a normalizer and pre-tokenizer. Note that
# BERT-specific normalizer and pre-tokenizer are used in this model.
tokenizer = Tokenizer(WordPiece())
tokenizer.normalizer = BertNormalizer(strip_accents=False)
tokenizer.pre_tokenizer = BertPreTokenizer()
# Train tokenizer model with subset of corpus.
trainer = WordPieceTrainer(vocab_size=self.vocab_size,
min_frequency=2,
show_progress=True,
limit_alphabet=self.limit_alphabet,
special_tokens=[self.unk_token] +
self.special_tokens,
continuing_subword_prefix='##')
tokenizer.train(trainer, [subset.name])
# Save trained vocabulary to an auxiliary output file.
vocab = afm.create()
tokenizer.model.save(os.path.dirname(vocab.name))
os.rename(os.path.join(os.path.dirname(vocab.name), 'vocab.txt'),
vocab.name)
return vocab
def train():
"""Source: https://huggingface.co/docs/tokenizers/pipeline"""
base = os.environ['DATA_ROOT']
corpus_path = base + 'MimicIII/Encounters/Text/'
bert_tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
# input to tokenizer.encode() goes through this pipeline:
# normalization, pre-tokenization, model, post-processing
bert_tokenizer.normalizer = normalizers.Sequence(
[NFD(), Lowercase(), StripAccents()])
bert_tokenizer.pre_tokenizer = Whitespace()
bert_tokenizer.post_processor = TemplateProcessing(
single="[CLS] $A [SEP]",
pair="[CLS] $A [SEP] $B:1 [SEP]:1",
special_tokens=[("[CLS]", 1), ("[SEP]", 2)])
files = [str(file) for file in Path(corpus_path).glob('*.txt')]
trainer = WordPieceTrainer(
vocab_size=30522,
show_progress=True,
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
bert_tokenizer.train(files, trainer)
os.mkdir('./Tokenizer')
bert_tokenizer.save("Tokenizer/tokenizer.json")
def train_tokenizer(input_dir: str,
save_path: str,
tokenizer_type: str = "BPE",
vocab_size: int = 52000):
"""
Trains a tokenizer on all the json files in `input_dir` and saves it to `save_path`
:param input_dir: input directory containing jsonl files
:param save_path: path to save tokenizer to
:param tokenizer_type: type of tokenizer to train.
:param vocab_size: int, size of tokenizer's vocab
:return:
"""
if tokenizer_type == "BPE":
model = models.BPE()
else:
raise NotImplementedError(
f'Tokenizer type {tokenizer_type} not implemented')
tokenizer = Tokenizer(model)
# Customize pre-tokenization and decoding
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=True)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.post_processor = processors.ByteLevel(trim_offsets=True)
tokenizer.normalizer = NFKC()
# And then train
trainer = trainers.BpeTrainer(
vocab_size=vocab_size, special_tokens=["<|endoftext|>", "<|padding|>"])
tokenizer.train_from_iterator(json_iterator(input_dir), trainer)
# And Save it
tokenizer.save(save_path, pretty=True)
print(f'Tokenizer saved at {save_path}')
def __init__(self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
unk_token: Optional[str] = "<unk>",
suffix: Optional[str] = "</w>",
dropout: Optional[float] = None):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE.from_files(vocab_file,
merges_file,
dropout=dropout,
unk_token=unk_token,
end_of_word_suffix=suffix))
else:
tokenizer = Tokenizer(BPE.empty())
tokenizer.normalizer = Sequence.new([NFKC.new(), Lowercase.new()])
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace.new()
tokenizer.decoder = decoders.BPEDecoder.new(suffix=suffix)
parameters = {
"model": "BPE",
"unk_token": unk_token,
"suffix": suffix,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def __init__(self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
unk_token: str = "<unk>",
replacement: str = "▁",
add_prefix_space: bool = True,
dropout: Optional[float] = None):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE.from_files(vocab_file,
merges_file,
dropout=dropout,
unk_token=unk_token))
else:
tokenizer = Tokenizer(BPE.empty())
tokenizer.add_special_tokens([unk_token])
tokenizer.normalizer = NFKC.new()
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace.new(
replacement=replacement, add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.Metaspace.new(
replacement=replacement, add_prefix_space=add_prefix_space)
parameters = {
"model": "SentencePieceBPE",
"unk_token": unk_token,
"replacement": replacement,
"add_prefix_space": add_prefix_space,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
vocab: Optional[Union[str, Dict[str, int]]] = None,
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int,
int]]]] = None,
unk_token: Union[str, AddedToken] = "<unk>",
replacement: str = "▁",
add_prefix_space: bool = True,
dropout: Optional[float] = None,
):
if vocab is not None and merges is not None:
tokenizer = Tokenizer(
BPE(vocab, merges, dropout=dropout, unk_token=unk_token))
else:
tokenizer = Tokenizer(BPE())
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
tokenizer.normalizer = NFKC()
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(
replacement=replacement, add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.Metaspace(
replacement=replacement, add_prefix_space=add_prefix_space)
parameters = {
"model": "SentencePieceBPE",
"unk_token": unk_token,
"replacement": replacement,
"add_prefix_space": add_prefix_space,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def test_normalize(self):
tokenizer = Tokenizer(BPE())
tokenizer.add_tokens(["my", "name", "is", "john", "pair"])
tokenizer.normalizer = Lowercase()
output = tokenizer.normalize("My Name Is John")
assert output == "my name is john"
def test_get_set_components(self):
toki = Tokenizer(models.BPE())
toki.normalizer = normalizers.NFC()
toki.pre_tokenizer = pre_tokenizers.ByteLevel()
toki.post_processor = processors.BertProcessing(("A", 0), ("B", 1))
toki.decoder = decoders.ByteLevel()
tokenizer = BaseTokenizer(toki)
assert isinstance(tokenizer.model, models.BPE)
assert isinstance(tokenizer.normalizer, normalizers.NFC)
assert isinstance(tokenizer.pre_tokenizer, pre_tokenizers.ByteLevel)
assert isinstance(tokenizer.post_processor, processors.BertProcessing)
assert isinstance(tokenizer.decoder, decoders.ByteLevel)
tokenizer.model = models.Unigram()
assert isinstance(tokenizer.model, models.Unigram)
tokenizer.normalizer = normalizers.NFD()
assert isinstance(tokenizer.normalizer, normalizers.NFD)
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
assert isinstance(tokenizer.pre_tokenizer, pre_tokenizers.Whitespace)
tokenizer.post_processor = processors.ByteLevel()
assert isinstance(tokenizer.post_processor, processors.ByteLevel)
tokenizer.decoder = decoders.WordPiece()
assert isinstance(tokenizer.decoder, decoders.WordPiece)
def get_daily_dialog_tokenizer(tokenizer_location=None):
'''
Get the daily dialog tokenizer. Trains a new one if no location is provided
:param tokenizer_location: Json containing information about the tokenizer.
:return:
'''
if tokenizer_location:
tokenizer = Tokenizer.from_file(tokenizer_location, )
tokenizer.enable_padding()
return tokenizer
else:
dataset_train = datasets.load_dataset("daily_dialog", split="train", )
utterances = [special_tokens["sep_token"].join(dialogue["dialog"]) for dialogue in dataset_train]
trainer = WordPieceTrainer(
vocab_size = 2048,
special_tokens = token_utils.special_tokens.values()
)
custom_tokenizer = Tokenizer(WordPiece(unk_token=special_tokens["unk_token"], ))
custom_tokenizer.normalizer = normalizers.Sequence([NFD(), Lowercase(), StripAccents()])
custom_tokenizer.pre_tokenizer = Whitespace()
custom_tokenizer.train_from_iterator(utterances, trainer, )
custom_tokenizer.enable_padding()
# Write every dialogue to file
location = './daily_dialog/'
custom_tokenizer.save(location + "tokenizer.json")
return custom_tokenizer
def __init__(
self,
vocab: Optional[str] = None,
replacement: str = "▁",
add_prefix_space: bool = True,
):
if vocab is not None:
# Let Unigram(..) fail if only one of them is None
tokenizer = Tokenizer(Unigram(vocab))
else:
tokenizer = Tokenizer(Unigram())
tokenizer.normalizer = normalizers.Sequence([
normalizers.Nmt(),
normalizers.NFKC(),
])
tokenizer.pre_tokenizer = pre_tokenizers.Sequence([
pre_tokenizers.WhitespaceSplit(),
pre_tokenizers.Metaspace(replacement=replacement,
add_prefix_space=add_prefix_space),
])
tokenizer.decoder = decoders.Metaspace(
replacement=replacement, add_prefix_space=add_prefix_space)
parameters = {
"model": "SentencePieceUnigram",
"replacement": replacement,
"add_prefix_space": add_prefix_space,
}
super().__init__(tokenizer, parameters)
def train_tokenizer(langs, dataset, vocab_size):
"""Train a tokenizer on given list of languages.
Reserves a special token for each language which is
[LANG] where LANG is the language tag. These are assigned
to tokens 5, 6, ..., len(langs) + 4.
"""
# Byte-pair encoding
tokenizer = Tokenizer(BPE(unk_token='[UNK]'))
# trainer
lang_tokens = ['[' + lang + ']' for lang in langs]
special_tokens = ['[MASK]', '[CLS]', '[SEP]', '[PAD]', '[UNK]'] + lang_tokens
trainer = BpeTrainer(
special_tokens=special_tokens,
vocab_size=vocab_size)
# normalise and pre tokenize
tokenizer.normalizer = Sequence([NFD(), Lowercase(), StripAccents()])
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
tokenizer.decoder = decoders.ByteLevel()
# create iterator and train
iterator = _MultilingualIterator(dataset, langs)
tokenizer.train_from_iterator(iterator, trainer)
# post process start/end tokens
tokenizer.post_processor = TemplateProcessing(
single="[CLS] $A [SEP]",
pair="[CLS] $A [SEP] $B:1 [SEP]:1",
special_tokens=[
("[CLS]", tokenizer.token_to_id("[CLS]")),
("[SEP]", tokenizer.token_to_id("[SEP]")),
], )
return tokenizer
def converted(self) -> Tokenizer:
tokenizer_info_str = "#version:"
token_suffix = "</w>"
vocab = self.original_tokenizer.encoder
merges = list(self.original_tokenizer.bpe_ranks.keys())
if tokenizer_info_str in merges[0][0]:
merges = merges[1:]
tokenizer = Tokenizer(
BPE(
vocab,
merges,
dropout=None,
unk_token=self.original_tokenizer.unk_token,
end_of_word_suffix=token_suffix,
))
tokenizer.normalizer = normalizers.BertNormalizer(lowercase=False,
strip_accents=False)
tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()
tokenizer.decoder = decoders.BPEDecoder(suffix=token_suffix)
tokenizer.post_processor = processors.BertProcessing(
sep=(self.original_tokenizer.sep_token,
self.original_tokenizer.sep_token_id),
cls=(self.original_tokenizer.cls_token,
self.original_tokenizer.cls_token_id),
)
return tokenizer
def get_tokenizer(args):
tokenizer = Tokenizer(models.BPE())
tokenizer.normalizer = Sequence(
[NFKC(), Replace('\r', ''),
Replace('\n', ' ')])
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
tokenizer.decoder = decoders.ByteLevel()
if os.path.isdir(args.tokenizer_dir):
vocab_fn = os.path.join(args.tokenizer_dir, 'vocab.json')
merge_fn = os.path.join(args.tokenizer_dir, 'merges.txt')
tokenizer.model = models.BPE.from_file(vocab_fn, merge_fn)
else:
os.makedirs(args.tokenizer_dir)
trainer = trainers.BpeTrainer(
vocab_size=args.vocab_size,
special_tokens=["[UNK]", "[PAD]", "[BOS]", "[EOS]"])
files = [
os.path.join(args.data_dir, split)
for split in ['train.json', 'val.json', 'test.json']
]
tokenizer.train(files=files, trainer=trainer)
tokenizer.model.save(args.tokenizer_dir)
return tokenizer
def main() -> None:
args = parse_args()
special_tokens = list(SPECIAL_TOKENS)
if args.reserved < len(special_tokens):
raise AssertionError(
f"number of reserved tokens should be more than number of f{len(special_tokens)}")
for i in range(len(special_tokens), args.reserved):
special_tokens.append(f"[unused{i:03d}]")
all_filenames = get_all_filenames(args.input)
# "C:\Users\demianmedich\data\wiki\20191120.en\pp_cased/"
tokenizer = Tokenizer(get_model(args.model))
tokenizer.normalizer = normalizers.Sequence([
NFKC(), StripAccents(), Lowercase()
])
tokenizer.pre_tokenizer = Whitespace()
trainer = WordPieceTrainer(
vocab_size=args.vocab_size,
special_tokens=special_tokens)
tokenizer.train(trainer, all_filenames)
model_files = tokenizer.model.save()
sys.exit(0)
def __init__(
self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
unk_token: Union[str, AddedToken] = "<unk>",
replacement: str = "▁",
add_prefix_space: bool = True,
no_consecutive_space: bool = True,
dropout: Optional[float] = None,
clean_text: bool = True,
handle_chinese_chars: bool = True,
separate_numbers: bool = True,
strip_accents: bool = True,
lowercase: bool = True,
wordpieces_prefix: str = "##",
special_chars: str = SPECIAL_CHARS,
zh_norm: bool = True,
):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE(vocab_file,
merges_file,
dropout=dropout,
unk_token=unk_token))
else:
tokenizer = Tokenizer(BPE())
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
tokenizer.normalizer = Sequence([
NFKC(),
BertNormalizer(clean_text=clean_text,
handle_chinese_chars=handle_chinese_chars,
separate_numbers=separate_numbers,
strip_accents=strip_accents,
lowercase=lowercase,
special_chars=special_chars,
zh_norm=zh_norm)
])
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(
replacement=replacement,
add_prefix_space=add_prefix_space,
no_consecutive_space=no_consecutive_space)
tokenizer.decoder = decoders.Metaspace(
replacement=replacement,
add_prefix_space=add_prefix_space,
no_consecutive_space=no_consecutive_space)
parameters = {
"model": "SentencePieceBPE",
"unk_token": unk_token,
"replacement": replacement,
"add_prefix_space": add_prefix_space,
"no_consecutive_space": no_consecutive_space,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
vocab_file,
delimiter,
lowercase,
unk_token,
eos_token,
add_eos=False,
add_double_eos=False,
normalization: Optional[str] = None,
):
try:
tokenizer = WordLevel.from_files(vocab_file, unk_token=unk_token)
tokenizer = Tokenizer(tokenizer)
except Exception:
raise ValueError(
"Unable to parse file {}. Unknown format. "
"If you tried to load a model saved through TransfoXLTokenizer,"
"please note they are not compatible.".format(vocab_file))
# Create the correct normalization path
normalizer = []
# Include unicode normalization
if normalization:
normalizer += [unicode_normalizer_from_str(normalization)]
# Include case normalization
if lowercase:
normalizer += [Lowercase()]
# Strip normalizer at the end
normalizer += [Strip(left=True, right=True)]
if len(normalizer) > 0:
tokenizer.normalizer = Sequence(
normalizer) if len(normalizer) > 1 else normalizer[0]
# Setup the splitter
tokenizer.pre_tokenizer = CharDelimiterSplit(
delimiter) if delimiter else WhitespaceSplit()
if add_double_eos:
tokenizer.post_processor = BertProcessing(
(eos_token, tokenizer.token_to_id(eos_token)),
(eos_token, tokenizer.token_to_id(eos_token)))
parameters = {
"model": "TransfoXLModel",
"add_eos": add_eos,
"add_double_eos": add_double_eos,
"unk_token": unk_token,
"eos_token": eos_token,
"delimiter": delimiter,
"lowercase": lowercase,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
unk_token: Optional[str] = "<unk>",
suffix: Optional[str] = "</w>",
dropout: Optional[float] = None,
unicode_normalizer: Optional[str] = None,
):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE.from_files(vocab_file,
merges_file,
dropout=dropout,
unk_token=unk_token,
end_of_word_suffix=suffix))
else:
tokenizer = Tokenizer(BPE.empty())
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
# OpenAI normalization is the same as Bert
normalizers += [BertNormalizer()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = BertPreTokenizer()
tokenizer.decoder = BPEDecoder(suffix=suffix)
parameters = {
"model": "BPE",
"unk_token": unk_token,
"suffix": suffix,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def test_strip_accents(self):
tokenizer = Tokenizer(BPE.empty())
tokenizer.normalizer = BertNormalizer(
strip_accents=True, lowercase=False, handle_chinese_chars=False, clean_text=False
)
output = tokenizer.normalize("Héllò")
assert output == "Hello"
def from_spm(filename: str):
try:
import sys
sys.path.append(".")
import sentencepiece_model_pb2 as model
except Exception:
raise Exception(
"You don't seem to have the required protobuf file, in order to use this function you need to run `pip install protobuf` and `wget https://raw.githubusercontent.com/google/sentencepiece/master/python/sentencepiece_model_pb2.py` for us to be able to read the intrinsics of your spm_file. `pip install sentencepiece` is not required."
)
m = model.ModelProto()
m.ParseFromString(open(filename, "rb").read())
precompiled_charsmap = m.normalizer_spec.precompiled_charsmap
vocab = [(piece.piece, piece.score) for piece in m.pieces]
unk_id = m.trainer_spec.unk_id
model_type = m.trainer_spec.model_type
if model_type != 1:
raise Exception(
"You're trying to run a `Unigram` model but you're file was trained with a different algorithm"
)
data = {"unk_id": unk_id, "vocab": vocab}
replacement = "▁"
add_prefix_space = True
out_vocab_filename = f"{filename}.json"
try:
with open(out_vocab_filename, "w") as f:
json.dump(data, f, indent=4)
tokenizer = Tokenizer(Unigram(out_vocab_filename))
finally:
os.remove(out_vocab_filename)
tokenizer.normalizer = normalizers.Precompiled(precompiled_charsmap)
tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
[
pre_tokenizers.WhitespaceSplit(),
pre_tokenizers.Metaspace(
replacement=replacement, add_prefix_space=add_prefix_space
),
]
)
tokenizer.decoder = decoders.Metaspace(
replacement=replacement, add_prefix_space=add_prefix_space
)
parameters = {
"model": "SentencePieceUnigram",
}
obj = BaseTokenizer.__new__(SentencePieceUnigramTokenizer, tokenizer, parameters)
BaseTokenizer.__init__(obj, tokenizer, parameters)
return obj
def test_clean_text(self):
tokenizer = Tokenizer(BPE())
tokenizer.normalizer = BertNormalizer(strip_accents=False,
lowercase=False,
handle_chinese_chars=False,
clean_text=True)
output = tokenizer.normalize("\ufeffHello")
assert output == "Hello"
def tokenize(dt, df):
from tokenizers import Tokenizer
from tokenizers.models import WordPiece
from tokenizers.pre_tokenizers import Whitespace
from tokenizers import normalizers
from tokenizers.normalizers import NFD, StripAccents
from tokenizers.processors import TemplateProcessing
from tokenizers.trainers import WordPieceTrainer
#print(df.head())
#print(df.query_text.head())
#print(df.query_text.to_list())
#exit(0)
data_source = get_data_source(dt)
token_file = Path(data_dir, data_source, 'tokenizer.json')
vocab_file = Path(data_dir, data_source, 'vocab.txt')
corpus_file = Path(data_dir, data_source, 'corpus.txt')
if vocab_file.is_file() and corpus_file.is_file():
print("corpus and token files already generated")
return 0
bert_tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
bert_tokenizer.normalizer = normalizers.Sequence([NFD(), StripAccents()])
bert_tokenizer.pre_tokenizer = Whitespace()
bert_tokenizer.post_processor = TemplateProcessing(
single="[CLS] $A [SEP]",
pair="[CLS] $A [SEP] $B:1 [SEP]:1",
special_tokens=[
("[CLS]", 1),
("[SEP]", 2),
],
)
trainer = WordPieceTrainer(
vocab_size=25000,
min_frequency=3,
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
#print(df.query_text.to_list())
bert_tokenizer.train_from_iterator(df.query_text.to_list(), trainer)
bert_tokenizer.save(str(token_file))
#bert_tokenizer.save_model(directory=data_dir,name='tokenizer')
df['range_idx'] = range(0, df.shape[0])
df['mean_rank_group'] = df.groupby(
['session_id'], sort=False)['range_idx'].transform(np.mean)
df['separate_column'] = df['range_idx'] < df['mean_rank_group']
df = df.groupby(['session_id', 'separate_column'],
as_index=False,
sort=False)['query_text'].agg(
' '.join).drop(columns='separate_column')
#df = df.groupby('session_id').agg({'query_text':' '.join}).reset_index()
df.query_text.to_csv(corpus_file, header=False, index=False)
with open(token_file) as token_f:
jdata = json.load(token_f)
with open(vocab_file, "w") as fd:
for k in jdata['model']['vocab'].keys():
print(k, file=fd)
def __init__(
self,
vocab_file: Optional[str] = None,
add_special_tokens: bool = True,
unk_token: str = "[UNK]",
sep_token: str = "[SEP]",
cls_token: str = "[CLS]",
clean_text: bool = True,
handle_chinese_chars: bool = True,
strip_accents: bool = True,
lowercase: bool = True,
wordpieces_prefix: str = "##",
):
if vocab_file is not None:
tokenizer = Tokenizer(
WordPiece.from_files(vocab_file, unk_token=unk_token))
else:
tokenizer = Tokenizer(WordPiece.empty())
tokenizer.add_special_tokens([unk_token, sep_token, cls_token])
tokenizer.normalizer = BertNormalizer(
clean_text=clean_text,
handle_chinese_chars=handle_chinese_chars,
strip_accents=strip_accents,
lowercase=lowercase,
)
tokenizer.pre_tokenizer = BertPreTokenizer()
if add_special_tokens and vocab_file is not None:
sep_token_id = tokenizer.token_to_id(sep_token)
if sep_token_id is None:
raise TypeError("sep_token not found in the vocabulary")
cls_token_id = tokenizer.token_to_id(cls_token)
if cls_token_id is None:
raise TypeError("cls_token not found in the vocabulary")
tokenizer.post_processor = BertProcessing(
(sep_token, sep_token_id), (cls_token, cls_token_id))
tokenizer.decoders = decoders.WordPiece(prefix=wordpieces_prefix)
parameters = {
"model": "BertWordPiece",
"add_special_tokens": add_special_tokens,
"unk_token": unk_token,
"sep_token": sep_token,
"cls_token": cls_token,
"clean_text": clean_text,
"handle_chinese_chars": handle_chinese_chars,
"strip_accents": strip_accents,
"lowercase": lowercase,
"wordpieces_prefix": wordpieces_prefix,
}
super().__init__(tokenizer, parameters)
def setup_tokenizer(_):
# Initialize a tokenizer
tokenizer = Tokenizer(models.BPE())
# Customize pre-tokenization and decoding
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=True)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.post_processor = processors.ByteLevel(trim_offsets=True)
normalizers = [NFKC()]
tokenizer.normalizer = Sequence(normalizers)
return tokenizer
def load_or_train_tokenizer(file_paths, tokenizer_mode_path):
'''
Tries to load saved text tokenizer
If there is none, trains the new tokenizer and saves is
'''
if not os.path.exists(tokenizer_mode_path):
print('Tokenizer model not found, training one')
from tokenizers.models import BPE
from tokenizers import Tokenizer
from tokenizers.decoders import ByteLevel as ByteLevelDecoder
from tokenizers.normalizers import NFKC, Sequence
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.trainers import BpeTrainer
tokenizer = Tokenizer(BPE())
tokenizer.normalizer = Sequence([
NFKC()
])
tokenizer.pre_tokenizer = ByteLevel()
tokenizer.decoder = ByteLevelDecoder()
trainer = BpeTrainer(
vocab_size=50000,
show_progress=True,
inital_alphabet=ByteLevel.alphabet(),
special_tokens=[
"<s>",
"<pad>",
"</s>",
"<unk>",
"<mask>"
]
)
tokenizer.train(file_paths, trainer)
if not os.path.exists(tokenizer_mode_path):
os.makedirs(tokenizer_mode_path)
tokenizer.model.save(tokenizer_mode_path, None)
print('Loading trained tokenizer model')
tokenizer = GPT2Tokenizer.from_pretrained(tokenizer_mode_path)
tokenizer.add_special_tokens({
'eos_token': '</s>',
'bos_token': '<s>',
'unk_token': '<unk>',
'pad_token': '<pad>',
'mask_token': '<mask>'
})
return tokenizer
def tokenizer_pipeline():
"""
specific pipeline for Cebuano Corpus tokenization
- Uses a Byte pair encoding (BPE) tokenizer
"""
tokenizer = Tokenizer(BPE())
# string normalization
tokenizer.normalizer = Sequence([NFD(), StripAccents(), Lowercase()])
tokenizer.pre_tokenizer = ByteLevel()
tokenizer.decoder = ByteLevelDecoder()
return tokenizer
def tokenize_corpus(
input_file: str,
output_file: str,
vocab_file: str,
unk_token: str = '<unk>',
control_tokens: List[str] = []):
r"""Tokenize corpus sentences through trained **WordPiece** model.
Arguments:
input_file (str): Input corpus file path.
output_file (str): Output file path.
vocab_file (str): Trained vocabulary file path.
unk_token (str): Unknown token in the vocabulary.
control_tokens (list): Control tokens in the vocabulary.
"""
# Create `WordPiece` model and add special tokens. Note that `unk_token`
# is also a special token.normalizer and pre-tokenizer.
tokenizer = Tokenizer(models.WordPiece(vocab_file, unk_token=unk_token))
tokenizer.add_special_tokens([unk_token] + control_tokens)
# Use BERT-specific normalizer, pre-tokenizer and **WordPiece** decoder.
tokenizer.normalizer = BertNormalizer(strip_accents=False)
tokenizer.pre_tokenizer = BertPreTokenizer()
tokenizer.decoder = decoders.WordPiece(prefix='##')
with open(input_file, 'r', encoding='utf-8') as src, \
open(output_file, 'w', encoding='utf-8') as dst:
# Count total lines in corpus.
total_lines = 0
for _ in src:
total_lines += 1
# Move the corpus file to first.
src.seek(0)
buffer = []
for line in tqdm.tqdm(src,
desc='[*] tokenize corpus',
total=total_lines):
buffer.append(line)
# Tokenize buffered sentences and write to `output_file`.
if len(buffer) > 10000:
for t in tokenizer.encode_batch(buffer):
dst.write(' '.join(t.tokens) + '\n')
buffer.clear()
# Process the remained buffer.
if buffer:
for t in tokenizer.encode_batch(buffer):
dst.write(' '.join(t.tokens) + '\n')
