def __init__(self, **kwargs):
super().__init__(**kwargs)
from tokenizers import Tokenizer, models, normalizers, pre_tokenizers
self.tokenizer = Tokenizer(
models.WordLevel(unk_token=self.unknown_token))
self.tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
if self.lower:
self.tokenizer.normalizer = normalizers.Lowercase()
def normalizer(self, proto):
list_normalizers = [normalizers.Replace("``", '"'), normalizers.Replace("''", '"')]
if not self.original_tokenizer.keep_accents:
list_normalizers.append(normalizers.NFKD())
list_normalizers.append(normalizers.StripAccents())
if self.original_tokenizer.do_lower_case:
list_normalizers.append(normalizers.Lowercase())
precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
list_normalizers.append(normalizers.Precompiled(precompiled_charsmap))
return normalizers.Sequence(list_normalizers)
def normalizer(self, proto):
list_normalizers = []
if self.original_tokenizer.do_lower_case:
list_normalizers.append(normalizers.Lowercase())
list_normalizers.append(normalizers.Strip())
precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
if precompiled_charsmap:
list_normalizers.append(
normalizers.Precompiled(precompiled_charsmap))
list_normalizers.append(normalizers.Replace(Regex(" {2,}"), " "))
return normalizers.Sequence(list_normalizers)
def __init__(
self,
replacement: str = "▁",
add_prefix_space: bool = True,
unk_token: Union[str, AddedToken] = "<unk>",
eos_token: Union[str, AddedToken] = "</s>",
pad_token: Union[str, AddedToken] = "<pad>",
):
self.special_tokens = {
"pad": {"id": 0, "token": pad_token},
"eos": {"id": 1, "token": eos_token},
"unk": {"id": 2, "token": unk_token},
}
self.special_tokens_list = [None] * len(self.special_tokens)
for token_dict in self.special_tokens.values():
self.special_tokens_list[token_dict["id"]] = token_dict["token"]
tokenizer = Tokenizer(Unigram())
tokenizer.normalizer = normalizers.Sequence(
[
normalizers.Nmt(),
normalizers.NFKC(),
normalizers.Replace(Regex(" {2,}"), " "),
normalizers.Lowercase(),
]
)
tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
[
pre_tokenizers.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space),
pre_tokenizers.Digits(individual_digits=True),
pre_tokenizers.Punctuation(),
]
)
tokenizer.decoder = decoders.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)
tokenizer.post_processor = TemplateProcessing(
single=f"$A {self.special_tokens['eos']['token']}",
special_tokens=[(self.special_tokens["eos"]["token"], self.special_tokens["eos"]["id"])],
)
parameters = {
"model": "SentencePieceUnigram",
"replacement": replacement,
"add_prefix_space": add_prefix_space,
}
super().__init__(tokenizer, parameters)
def train_tokenizer() -> Tuple[tokenizers.Tokenizer, Generator, int]:
tokenizer = tokenizers.Tokenizer(models.WordPiece(unk_token="<unk>"))
tokenizer.decoder = decoders.WordPiece()
tokenizer.normalizer = normalizers.Sequence([
normalizers.NFD(), # NFD unicode normalizer
normalizers.Lowercase(),
normalizers.StripAccents()
])
tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Sequence([
pre_tokenizers.Whitespace(),
pre_tokenizers.Digits(individual_digits=False)
])
tokenizer.post_processor = processors.TemplateProcessing(
single="$A </s>",
pair="$A </s> [SEP] <s> $B:1",
special_tokens=[("[SEP]", 1), ("<s>", 2), ("</s>", 3)])
dataset = datasets.load_dataset("wikitext",
"wikitext-103-raw-v1",
split="test")
def batch_iterator(batch_size=1000):
for i in range(0, len(dataset), batch_size):
yield dataset[i:i + batch_size]["text"]
tokenizer.train_from_iterator(
batch_iterator(),
trainer=trainers.WordPieceTrainer(
vocab_size=10000, special_tokens=["<unk>", "[SEP]", "<s>",
"</s>"]))
def generator():
for record in dataset:
if record['text'].strip() != '':
for sentence in sent_tokenizer(record['text']):
yield sentence
data = tf.data.Dataset.from_generator(generator,
output_signature=(tf.TensorSpec(
shape=(None), dtype=tf.string)))
data = data.map(tf.strings.strip,
num_parallel_calls=tf.data.experimental.AUTOTUNE)
return tokenizer, data
def converted(self) -> Tokenizer:
vocab = self.original_tokenizer.encoder
merges = list(self.original_tokenizer.bpe_ranks.keys())
unk_token = self.original_tokenizer.unk_token
tokenizer = Tokenizer(
BPE(
vocab=vocab,
merges=merges,
dropout=None,
continuing_subword_prefix="",
end_of_word_suffix="</w>",
fuse_unk=False,
unk_token=str(unk_token),
))
tokenizer.normalizer = normalizers.Sequence([
normalizers.NFC(),
normalizers.Replace(Regex(r"\s+"), " "),
normalizers.Lowercase()
])
tokenizer.pre_tokenizer = pre_tokenizers.Sequence([
pre_tokenizers.Split(
Regex(
r"""'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+"""
),
behavior="removed",
invert=True,
),
pre_tokenizers.ByteLevel(add_prefix_space=False),
])
tokenizer.decoder = decoders.ByteLevel()
# Hack to have a ByteLevel and TemplaceProcessor
tokenizer.post_processor = processors.RobertaProcessing(
sep=(self.original_tokenizer.eos_token,
self.original_tokenizer.eos_token_id),
cls=(self.original_tokenizer.bos_token,
self.original_tokenizer.bos_token_id),
add_prefix_space=False,
trim_offsets=False,
)
return tokenizer
def test_train_parallelism_with_custom_pretokenizer(self, train_files):
class GoodCustomPretok:
def split(self, n, normalized):
# Here we just test that we can return a List[NormalizedString], it
# does not really make sense to return twice the same otherwise
return [normalized, normalized]
def pre_tokenize(self, pretok):
pretok.split(self.split)
custom = pre_tokenizers.PreTokenizer.custom(GoodCustomPretok())
bpe_tokenizer = Tokenizer(models.BPE())
bpe_tokenizer.normalizer = normalizers.Lowercase()
bpe_tokenizer.pre_tokenizer = custom
if "TOKENIZERS_PARALLELISM" in os.environ:
del os.environ["TOKENIZERS_PARALLELISM"]
trainer = trainers.BpeTrainer(special_tokens=["<unk>"], show_progress=False)
bpe_tokenizer.train([train_files["small"]], trainer=trainer)
def train_tokenizer() -> tokenizers.Tokenizer:
tokenizer = tokenizers.Tokenizer(models.WordPiece(unk_token="<unk>"))
tokenizer.decoder = decoders.WordPiece()
tokenizer.normalizer = normalizers.Sequence([
normalizers.NFD(), # NFD unicode normalizer
normalizers.Lowercase(),
normalizers.StripAccents()
])
tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Sequence([
pre_tokenizers.Whitespace(),
pre_tokenizers.Digits(individual_digits=False)
])
tokenizer.post_processor = processors.TemplateProcessing(
single="$A </s>",
pair="$A </s> [SEP] <s> $B:1",
special_tokens=[("[SEP]", 1), ("<s>", 2), ("</s>", 3)])
# dataset = datasets.load_dataset("wikitext", "wikitext-103-raw-v1", split="train+test+validation")
dataset = datasets.load_dataset("wikitext",
"wikitext-103-raw-v1",
split="validation")
def batch_iterator(batch_size=1000):
for i in range(0, len(dataset), batch_size):
yield dataset[i:i + batch_size]["text"]
tokenizer.train_from_iterator(
batch_iterator(),
trainer=trainers.WordPieceTrainer(
vocab_size=10000, special_tokens=["<unk>", "[SEP]", "<s>",
"</s>"]))
def generator():
for record in dataset:
if record['text'].strip() != '':
yield record['text']
return tokenizer, generator
import datasets
from tokenizers import normalizers, pre_tokenizers, Tokenizer, models, trainers
# Build a tokenizer
bpe_tokenizer = Tokenizer(models.BPE())
bpe_tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
bpe_tokenizer.normalizer = normalizers.Lowercase()
# Initialize a dataset
dataset = datasets.load_dataset("wikitext", "wikitext-103-raw-v1")
# Build an iterator over this dataset
def batch_iterator():
batch_length = 1000
for i in range(0, len(dataset["train"]), batch_length):
yield dataset["train"][i:i + batch_length]["text"]
# And finally train
bpe_tokenizer.train_from_iterator(batch_iterator(),
length=len(dataset["train"]))
