def get_recurrent_tokenizer(vocab,
max_context_tokens,
unk_token,
pad_token,
device="cpu"):
"""
Return a tokenizer to be used with recurrent-based models
"""
question_tokenizer = Tokenizer(WordLevel(vocab, unk_token=unk_token))
question_tokenizer.normalizer = Sequence(
[StripAccents(), Lowercase(), Strip()])
question_tokenizer.pre_tokenizer = PreSequence(
[Whitespace(), Punctuation()])
question_tokenizer.enable_padding(direction="right",
pad_id=vocab[pad_token],
pad_type_id=1,
pad_token=pad_token)
context_tokenizer = Tokenizer(WordLevel(vocab, unk_token=unk_token))
context_tokenizer.normalizer = Sequence(
[StripAccents(), Lowercase(), Strip()])
context_tokenizer.pre_tokenizer = PreSequence(
[Whitespace(), Punctuation()])
context_tokenizer.enable_padding(
direction="right",
pad_id=vocab[pad_token],
pad_type_id=1,
pad_token=pad_token,
)
context_tokenizer.enable_truncation(max_context_tokens)
return RecurrentSquadTokenizer(question_tokenizer,
context_tokenizer,
device=device)
def train_tokenizer(langs, dataset, vocab_size):
"""Train a tokenizer on given list of languages.
Reserves a special token for each language which is
[LANG] where LANG is the language tag. These are assigned
to tokens 5, 6, ..., len(langs) + 4.
"""
# Byte-pair encoding
tokenizer = Tokenizer(BPE(unk_token='[UNK]'))
# trainer
lang_tokens = ['[' + lang + ']' for lang in langs]
special_tokens = ['[MASK]', '[CLS]', '[SEP]', '[PAD]', '[UNK]'] + lang_tokens
trainer = BpeTrainer(
special_tokens=special_tokens,
vocab_size=vocab_size)
# normalise and pre tokenize
tokenizer.normalizer = Sequence([NFD(), Lowercase(), StripAccents()])
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
tokenizer.decoder = decoders.ByteLevel()
# create iterator and train
iterator = _MultilingualIterator(dataset, langs)
tokenizer.train_from_iterator(iterator, trainer)
# post process start/end tokens
tokenizer.post_processor = TemplateProcessing(
single="[CLS] $A [SEP]",
pair="[CLS] $A [SEP] $B:1 [SEP]:1",
special_tokens=[
("[CLS]", tokenizer.token_to_id("[CLS]")),
("[SEP]", tokenizer.token_to_id("[SEP]")),
], )
return tokenizer
def __init__(self):
self.tokenizer = Tokenizer(BPE())
self.tokenizer.normalizer = Sequence([
NFKC()
])
self.tokenizer.pre_tokenizer = ByteLevel()
self.tokenizer.decoder = ByteLevelDecoder()
def __init__(self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
unk_token: Optional[str] = "<unk>",
suffix: Optional[str] = "</w>",
dropout: Optional[float] = None):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE.from_files(vocab_file,
merges_file,
dropout=dropout,
unk_token=unk_token,
end_of_word_suffix=suffix))
else:
tokenizer = Tokenizer(BPE.empty())
tokenizer.normalizer = Sequence.new([NFKC.new(), Lowercase.new()])
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace.new()
tokenizer.decoder = decoders.BPEDecoder.new(suffix=suffix)
parameters = {
"model": "BPE",
"unk_token": unk_token,
"suffix": suffix,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
load_from: str = None,
vocab_size: int = 10000,
max_example_len: int = 128,
batch_size: int = 16,
num_stopwords: int = 250,
mask_output_len: int = 4,
):
self.char_dict: Dict[str, int] = {}
self.char_rev: Dict[int, str] = {}
self.token_dict: Dict[str, int] = {}
self.token_rev: Dict[str, int] = {}
self.vocab_size = vocab_size
self.max_example_len = max_example_len
self.batch_size = batch_size
self.num_stopwords = num_stopwords
self.mask_output_len = mask_output_len
self.tokenizer_fit = False
self.tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
self.tokenizer.pre_tokenizer = Whitespace()
self.tokenizer.normalizer = Sequence(
[NFD(), Lowercase(), StripAccents()])
self.tok_trainer = BpeTrainer(special_tokens=["[UNK]", "[MASK]"],
vocab_size=self.vocab_size)
if load_from:
self._load(load_from)
def __init__(
self,
path_src,
path_tgt,
path_tokenizer,
path_root: Optional[str] = '',
):
self.path_src = path_root + path_src
self.path_tgt = path_root + path_tgt
self.len = 0
self.max_len = 512
self.tokenizer = Tokenizer(
BPE(
path_root + path_tokenizer + 'vocab.json',
path_root + path_tokenizer + 'merges.txt',
))
self.tokenizer.normalizer = Sequence([NFKC(), Lowercase()])
with open(self.path_src, 'r+') as f:
lines_src = f.readlines()
with open(self.path_tgt, 'r+') as f:
lines_tgt = f.readlines()
self.len = len(lines_src)
self.example = list(zip(lines_src, lines_tgt))
def __init__(self,
vocab_size=25000,
min_freq=5,
lang="en",
files=[None, None]) -> None:
"""
Args:
vocab_size: (int)
min_freq: minimum frequency
lang:
files: (List[str]) ["vocab.json", "merge.txt"]
"""
super(BPETokenizer, self).__init__()
self.tokenizer = Tokenizer(BPE(files[0], files[1]))
self.lang = lang
self.trainer = BpeTrainer(vocab_size=vocab_size,
min_frequency=min_freq,
special_tokens=["[PAD]", "[SEP]"],
initial_alphabet=ByteLevel.alphabet())
# https://huggingface.co/docs/tokenizers/python/latest/components.html#normalizers
self.tokenizer.normalizer = Sequence([NFKC(), Lowercase()])
# https://huggingface.co/docs/tokenizers/python/latest/components.html#pre-tokenizers
self.tokenizer.pre_tokenizer = ByteLevel()
self.tokenizer.decoder = ByteLevelDecoder()
def get_tokenizer(args):
tokenizer = Tokenizer(models.BPE())
tokenizer.normalizer = Sequence(
[NFKC(), Replace('\r', ''),
Replace('\n', ' ')])
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
tokenizer.decoder = decoders.ByteLevel()
if os.path.isdir(args.tokenizer_dir):
vocab_fn = os.path.join(args.tokenizer_dir, 'vocab.json')
merge_fn = os.path.join(args.tokenizer_dir, 'merges.txt')
tokenizer.model = models.BPE.from_file(vocab_fn, merge_fn)
else:
os.makedirs(args.tokenizer_dir)
trainer = trainers.BpeTrainer(
vocab_size=args.vocab_size,
special_tokens=["[UNK]", "[PAD]", "[BOS]", "[EOS]"])
files = [
os.path.join(args.data_dir, split)
for split in ['train.json', 'val.json', 'test.json']
]
tokenizer.train(files=files, trainer=trainer)
tokenizer.model.save(args.tokenizer_dir)
return tokenizer
def __init__(
self,
vocab: Optional[Union[str, Dict[str, int]]] = None,
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int,
int]]]] = None,
add_prefix_space: bool = False,
lowercase: bool = False,
dropout: Optional[float] = None,
unicode_normalizer: Optional[str] = None,
continuing_subword_prefix: Optional[str] = None,
end_of_word_suffix: Optional[str] = None,
trim_offsets: bool = False,
):
if vocab is not None and merges is not None:
tokenizer = Tokenizer(
BPE(
vocab,
merges,
dropout=dropout,
continuing_subword_prefix=continuing_subword_prefix or "",
end_of_word_suffix=end_of_word_suffix or "",
))
else:
tokenizer = Tokenizer(BPE())
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
if lowercase:
normalizers += [Lowercase()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.post_processor = processors.ByteLevel(
trim_offsets=trim_offsets)
parameters = {
"model": "ByteLevelBPE",
"add_prefix_space": add_prefix_space,
"lowercase": lowercase,
"dropout": dropout,
"unicode_normalizer": unicode_normalizer,
"continuing_subword_prefix": continuing_subword_prefix,
"end_of_word_suffix": end_of_word_suffix,
"trim_offsets": trim_offsets,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
vocab_file,
delimiter,
lowercase,
unk_token,
eos_token,
add_eos=False,
add_double_eos=False,
normalization: Optional[str] = None,
):
try:
tokenizer = WordLevel(vocab_file, unk_token=unk_token)
tokenizer = Tokenizer(tokenizer)
except Exception:
raise ValueError(
"Unable to parse file {}. Unknown format. "
"If you tried to load a model saved through TransfoXLTokenizer,"
"please note they are not compatible.".format(vocab_file))
# Create the correct normalization path
normalizer = []
# Include unicode normalization
if normalization:
normalizer += [unicode_normalizer_from_str(normalization)]
# Include case normalization
if lowercase:
normalizer += [Lowercase()]
# Strip normalizer at the end
normalizer += [Strip(left=True, right=True)]
if len(normalizer) > 0:
tokenizer.normalizer = Sequence(
normalizer) if len(normalizer) > 1 else normalizer[0]
# Setup the splitter
tokenizer.pre_tokenizer = CharDelimiterSplit(
delimiter) if delimiter else WhitespaceSplit()
if add_double_eos:
tokenizer.post_processor = BertProcessing(
(eos_token, tokenizer.token_to_id(eos_token)),
(eos_token, tokenizer.token_to_id(eos_token)))
parameters = {
"model": "TransfoXLModel",
"add_eos": add_eos,
"add_double_eos": add_double_eos,
"unk_token": unk_token,
"eos_token": eos_token,
"delimiter": delimiter,
"lowercase": lowercase,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
unk_token: Union[str, AddedToken] = "<unk>",
replacement: str = "▁",
add_prefix_space: bool = True,
no_consecutive_space: bool = True,
dropout: Optional[float] = None,
clean_text: bool = True,
handle_chinese_chars: bool = True,
separate_numbers: bool = True,
strip_accents: bool = True,
lowercase: bool = True,
wordpieces_prefix: str = "##",
special_chars: str = SPECIAL_CHARS,
zh_norm: bool = True,
):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE(vocab_file,
merges_file,
dropout=dropout,
unk_token=unk_token))
else:
tokenizer = Tokenizer(BPE())
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
tokenizer.normalizer = Sequence([
NFKC(),
BertNormalizer(clean_text=clean_text,
handle_chinese_chars=handle_chinese_chars,
separate_numbers=separate_numbers,
strip_accents=strip_accents,
lowercase=lowercase,
special_chars=special_chars,
zh_norm=zh_norm)
])
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(
replacement=replacement,
add_prefix_space=add_prefix_space,
no_consecutive_space=no_consecutive_space)
tokenizer.decoder = decoders.Metaspace(
replacement=replacement,
add_prefix_space=add_prefix_space,
no_consecutive_space=no_consecutive_space)
parameters = {
"model": "SentencePieceBPE",
"unk_token": unk_token,
"replacement": replacement,
"add_prefix_space": add_prefix_space,
"no_consecutive_space": no_consecutive_space,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
add_prefix_space: bool = False,
lowercase: bool = False,
dropout: Optional[float] = None,
unicode_normalizer: Optional[str] = None,
continuing_subword_prefix: Optional[str] = None,
end_of_word_suffix: Optional[str] = None,
):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE.from_files(
vocab_file,
merges_file,
dropout=dropout,
continuing_subword_prefix=continuing_subword_prefix or "",
end_of_word_suffix=end_of_word_suffix or "",
))
else:
tokenizer = Tokenizer(BPE.empty())
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
if lowercase:
normalizers += [Lowercase()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.ByteLevel()
parameters = {
"model": "ByteLevelBPE",
"add_prefix_space": add_prefix_space,
"lowercase": lowercase,
"dropout": dropout,
"unicode_normalizer": unicode_normalizer,
"continuing_subword_prefix": continuing_subword_prefix,
"end_of_word_suffix": end_of_word_suffix,
}
super().__init__(tokenizer, parameters)
def setup_tokenizer(_):
# Initialize a tokenizer
tokenizer = Tokenizer(models.BPE())
# Customize pre-tokenization and decoding
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=True)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.post_processor = processors.ByteLevel(trim_offsets=True)
normalizers = [NFKC()]
tokenizer.normalizer = Sequence(normalizers)
return tokenizer
def normalizer(self, proto):
normalizers = [Replace("``", '"'), Replace("''", '"')]
if not self.original_tokenizer.keep_accents:
normalizers.append(NFKD())
normalizers.append(StripAccents())
if self.original_tokenizer.do_lower_case:
normalizers.append(Lowercase())
precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
normalizers.append(Precompiled(precompiled_charsmap))
return Sequence(normalizers)
def load_or_train_tokenizer(file_paths, tokenizer_mode_path):
'''
Tries to load saved text tokenizer
If there is none, trains the new tokenizer and saves is
'''
if not os.path.exists(tokenizer_mode_path):
print('Tokenizer model not found, training one')
from tokenizers.models import BPE
from tokenizers import Tokenizer
from tokenizers.decoders import ByteLevel as ByteLevelDecoder
from tokenizers.normalizers import NFKC, Sequence
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.trainers import BpeTrainer
tokenizer = Tokenizer(BPE())
tokenizer.normalizer = Sequence([
NFKC()
])
tokenizer.pre_tokenizer = ByteLevel()
tokenizer.decoder = ByteLevelDecoder()
trainer = BpeTrainer(
vocab_size=50000,
show_progress=True,
inital_alphabet=ByteLevel.alphabet(),
special_tokens=[
"<s>",
"<pad>",
"</s>",
"<unk>",
"<mask>"
]
)
tokenizer.train(file_paths, trainer)
if not os.path.exists(tokenizer_mode_path):
os.makedirs(tokenizer_mode_path)
tokenizer.model.save(tokenizer_mode_path, None)
print('Loading trained tokenizer model')
tokenizer = GPT2Tokenizer.from_pretrained(tokenizer_mode_path)
tokenizer.add_special_tokens({
'eos_token': '</s>',
'bos_token': '<s>',
'unk_token': '<unk>',
'pad_token': '<pad>',
'mask_token': '<mask>'
})
return tokenizer
def tokenizer_pipeline():
"""
specific pipeline for Cebuano Corpus tokenization
- Uses a Byte pair encoding (BPE) tokenizer
"""
tokenizer = Tokenizer(BPE())
# string normalization
tokenizer.normalizer = Sequence([NFD(), StripAccents(), Lowercase()])
tokenizer.pre_tokenizer = ByteLevel()
tokenizer.decoder = ByteLevelDecoder()
return tokenizer
def __init__(
self,
vocab_file: Optional[str] = None,
unk_token: Union[str, AddedToken] = "[UNK]",
pad_token: Union[str, AddedToken] = "[PAD]",
mask_token: Union[str, AddedToken] = "[MASK]",
lowercase: bool = False,
unicode_normalizer: Optional[str] = None,
):
if vocab_file is not None:
logging.info(f"Initiating tokenizer at {vocab_file}")
tokenizer = Tokenizer(
WordLevel(vocab=vocab_file, unk_token=unk_token))
else:
tokenizer = Tokenizer(WordLevel(unk_token=unk_token))
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
if tokenizer.token_to_id(str(pad_token)) is not None:
tokenizer.add_special_tokens([str(pad_token)])
if tokenizer.token_to_id(str(mask_token)) is not None:
tokenizer.add_special_tokens([str(mask_token)])
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
if lowercase:
normalizers += [Lowercase()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
parameters = {
"model": "WordLevel",
"unk_token": unk_token,
"pad_token": pad_token,
"mask_token": mask_token,
"lowercase": lowercase,
"unicode_normalizer": unicode_normalizer,
}
super().__init__(tokenizer, parameters)
def get_tokenizer(self, tokenizer_dir):
tokenizer = Tokenizer(models.BPE())
tokenizer.normalizer = Sequence(
[NFKC(), Replace('\r', ''),
Replace('\n', ' ')])
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
tokenizer.decoder = decoders.ByteLevel()
vocab_fn = os.path.join(tokenizer_dir, 'vocab.json')
merge_fn = os.path.join(tokenizer_dir, 'merges.txt')
tokenizer.model = models.BPE.from_file(vocab_fn, merge_fn)
tokenizer.add_special_tokens(['[UNK]', '[PAD]', '[BOS]', '[EOS]'])
return tokenizer
def __init__(
self,
vocab_file,
delimiter,
lowercase,
unk_token,
eos_token,
add_eos=False,
add_double_eos=False,
normalization: Optional[str] = None,
):
tokenizer = WordLevel.from_files(vocab_file, unk_token=unk_token)
tokenizer = Tokenizer(tokenizer)
# Create the correct normalization path
normalizer = []
# Include unicode normalization
if normalization:
normalizer += [unicode_normalizer_from_str(normalization)]
# Include case normalization
if lowercase:
normalizer += [Lowercase()]
if len(normalizer) > 0:
tokenizer.normalizer = Sequence(
normalizer) if len(normalizer) > 1 else normalizer[0]
# Setup the splitter
tokenizer.pre_tokenizer = CharDelimiterSplit(
delimiter) if delimiter else WhitespaceSplit()
if add_double_eos:
tokenizer.post_processor = BertProcessing(
(eos_token, tokenizer.token_to_id(eos_token)),
(eos_token, tokenizer.token_to_id(eos_token)))
parameters = {
"model": "TransfoXLModel",
"add_eos": add_eos,
"add_double_eos": add_double_eos,
"unk_token": unk_token,
"eos_token": eos_token,
"delimiter": delimiter,
"lowercase": lowercase,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
unk_token: Optional[str] = "<unk>",
suffix: Optional[str] = "</w>",
dropout: Optional[float] = None,
unicode_normalizer: Optional[str] = None,
):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE.from_files(vocab_file,
merges_file,
dropout=dropout,
unk_token=unk_token,
end_of_word_suffix=suffix))
else:
tokenizer = Tokenizer(BPE.empty())
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
# OpenAI normalization is the same as Bert
normalizers += [BertNormalizer()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = BertPreTokenizer()
tokenizer.decoder = BPEDecoder(suffix=suffix)
parameters = {
"model": "BPE",
"unk_token": unk_token,
"suffix": suffix,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def train_tokenizer(sentences: List[str],
serialize_path: str = "",
vocab_size: int = 6000) -> Tokenizer:
bert_tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
bert_tokenizer.normalizer = Sequence([NFD(), Lowercase(), StripAccents()])
bert_tokenizer.pre_tokenizer = Whitespace()
bert_tokenizer.post_processor = TemplateProcessing(
single="[CLS] $A [SEP]",
pair="[CLS] $A [SEP] $B:1 [SEP]:1",
special_tokens=[
("[CLS]", 1),
("[SEP]", 2),
],
)
trainer = WordPieceTrainer(
vocab_size=vocab_size,
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
bert_tokenizer.train_from_iterator(sentences, trainer=trainer)
if serialize_path:
bert_tokenizer.save_model(serialize_path)
return bert_tokenizer
def main(args):
if args.do_train:
# Initialize a tokenizer
files = get_smi_files(args.training_files)
print("Training BPE tokenizer using the following files:{}".format(
files))
tokenizer = Tokenizer(models.BPE(unk_token="<unk>"))
tokenizer.enable_padding(pad_id=args.vocab_size + 2,
pad_token="<pad>",
length=args.pad_len)
tokenizer.enable_truncation(max_length=args.pad_len,
strategy='only_first')
tokenizer.normalizer = Sequence([NFKC()])
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
add_prefix_space=False)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.post_processor = processors.ByteLevel(trim_offsets=True)
# Train the tokenizer
trainer = trainers.BpeTrainer(show_progress=True,
vocab_size=args.vocab_size,
min_frequency=args.min_frequency)
tokenizer.train(files, trainer=trainer)
tokenizer.add_tokens(["<start>", "<end>"])
tokenizer.save(os.path.join('tokenizers', args.tokenizer_name),
pretty=True)
print("Trained vocab size: {}".format(tokenizer.get_vocab_size()))
if args.do_test:
# Test the tokenizer
tokenizer = Tokenizer.from_file(
os.path.join('tokenizers', args.tokenizer_name))
print("Testing with SMILES String: {}".format(args.test_string))
encoding = tokenizer.encode(args.test_string)
print("Encoded string: {}".format(encoding.tokens))
print(encoding.ids)
decoded = tokenizer.decode(encoding.ids)
print("Decoded string: {}".format(decoded))
def normalizer(self, proto):
precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
return Sequence(
[Precompiled(precompiled_charsmap),
Replace(Regex(" {2,}"), " ")])
# name = mention + concept
names = unique([mentions + list(concepts)], verbose=False) - stop_words
# names = unique([mentions + list(concepts)], verbose=False)
print(f"Unique names: {len(names)}\n")
name_words = {n: " ".join(split_to_words(n)) for n in names}
with open(f"{proc_path}/names.txt", "w") as f:
f.write("\n".join(list(name_words.values())))
# f.write("\n".join(words))
tokenizer = Tokenizer(BPE())
tokenizer.normalizer = Sequence([
# NFKC(),
Lowercase()
])
tokenizer.pre_tokenizer = ByteLevel()
tokenizer.decoder = ByteLevelDecoder()
trainer = BpeTrainer(vocab_size=int(vocab_size), show_progress=True)
tokenizer.train(trainer, [f"{proc_path}/names.txt"])
print("Trained vocab size: {}".format(tokenizer.get_vocab_size()))
tokenizer.model.save(proc_path)
tokenizer.model = BPE.from_file(f'{proc_path}/vocab.json',
f'{proc_path}/merges.txt')
with open(f"{proc_path}/vocab.json", "r") as f:
bpe_vocab = json.load(f)
def test_can_make_sequences(self):
normalizer = Sequence([Lowercase(), Strip()])
output = normalizer.normalize_str(" HELLO ")
assert output == "hello"
def test_instantiate(self):
assert isinstance(Sequence([]), Normalizer)
assert isinstance(Sequence([]), Sequence)
assert isinstance(pickle.loads(pickle.dumps(Sequence([]))), Sequence)
for v in words:
print(v)
# Start vocabulary with all standard special tokens. (PAD=0!)
vocab = {}
for special_token in ["[PAD]", "[CLS]", "[SEP]", "[UNK]", "[MASK]", "[BOS]", "[EOS]"]:
vocab[special_token] = len(vocab)
# Add other words - if not already present.
for w in words:
if w not in vocab:
vocab[w] = len(vocab)
print(vocab)
# New tokenizer.
init_tokenizer = BertWordPieceTokenizer(vocab=vocab)
init_tokenizer.normalizer = Sequence([Replace("(", " ( "), Replace(")", " ) "), BertNormalizer()])
init_tokenizer.pre_tokenizer = Whitespace()
#init_tokenizer.pad_token_id = vocab["[PAD]"]
#print("Created tokenizer: ", init_tokenizer)
# Save the created tokenizer.
init_tokenizer.save(decoder_tokenizer_path)
print("Tokenizer saved to: ", decoder_tokenizer_path)
# Load from tokenizer file.
tokenizer = PreTrainedTokenizerFast(tokenizer_file=decoder_tokenizer_path)
tokenizer.add_special_tokens({'pad_token': '[PAD]', 'cls_token': '[CLS]', 'sep_token': '[SEP]',
'unk_token': '[UNK]', 'mask_token': '[MASK]', 'bos_token': '[BOS]', 'eos_token': '[EOS]'
})
print(f"\nLoaded tokenizer vocabulary ({len(tokenizer.get_vocab())}):\n" + "-"*50)
def prepare_tokenizer(self):
tokenizer = Tokenizer(BPE(unk_token="<unk>"))
tokenizer.normalizer = Sequence([Lowercase(), NFKC()])
tokenizer.pre_tokenizer = ByteLevelPreTokenizer()
tokenizer.decoder = ByteLevelDecoder()
return tokenizer
def test_can_make_sequences(self):
tokenizer = Tokenizer(BPE.empty())
tokenizer.normalizer = Sequence([Lowercase(), Strip()])
output = tokenizer.normalize(" HELLO ")
assert output == "hello"
def prepare_tokenizer(self):
tokenizer = Tokenizer(WordPiece(unk_token='[UNK]'))
tokenizer.pre_tokenizer = Whitespace()
tokenizer.normalizer = Sequence([Lowercase(), NFKC()])
return tokenizer
