def prepare_tokenizer(self):
tokenizer = Tokenizer(WordPiece(unk_token='[UNK]'))
tokenizer.pre_tokenizer = Whitespace()
tokenizer.normalizer = Sequence([Lowercase(), NFKC()])
return tokenizer
def test_multiprocessing_with_parallelism(self):
tokenizer = Tokenizer(BPE())
multiprocessing_with_parallelism(tokenizer, False)
multiprocessing_with_parallelism(tokenizer, True)
def __init__(
self,
vocab: Optional[Union[str, Dict[str, int]]] = None,
unk_token: Union[str, AddedToken] = "[UNK]",
sep_token: Union[str, AddedToken] = "[SEP]",
cls_token: Union[str, AddedToken] = "[CLS]",
pad_token: Union[str, AddedToken] = "[PAD]",
mask_token: Union[str, AddedToken] = "[MASK]",
clean_text: bool = True,
handle_chinese_chars: bool = True,
strip_accents: Optional[bool] = None,
lowercase: bool = True,
wordpieces_prefix: str = "##",
):
if vocab is not None:
tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(unk_token)))
else:
tokenizer = Tokenizer(WordPiece(unk_token=str(unk_token)))
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
if tokenizer.token_to_id(str(sep_token)) is not None:
tokenizer.add_special_tokens([str(sep_token)])
if tokenizer.token_to_id(str(cls_token)) is not None:
tokenizer.add_special_tokens([str(cls_token)])
if tokenizer.token_to_id(str(pad_token)) is not None:
tokenizer.add_special_tokens([str(pad_token)])
if tokenizer.token_to_id(str(mask_token)) is not None:
tokenizer.add_special_tokens([str(mask_token)])
tokenizer.normalizer = BertNormalizer(
clean_text=clean_text,
handle_chinese_chars=handle_chinese_chars,
strip_accents=strip_accents,
lowercase=lowercase,
)
tokenizer.pre_tokenizer = BertPreTokenizer()
if vocab is not None:
sep_token_id = tokenizer.token_to_id(str(sep_token))
if sep_token_id is None:
raise TypeError("sep_token not found in the vocabulary")
cls_token_id = tokenizer.token_to_id(str(cls_token))
if cls_token_id is None:
raise TypeError("cls_token not found in the vocabulary")
tokenizer.post_processor = BertProcessing(
(str(sep_token), sep_token_id), (str(cls_token), cls_token_id))
tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix)
parameters = {
"model": "BertWordPiece",
"unk_token": unk_token,
"sep_token": sep_token,
"cls_token": cls_token,
"pad_token": pad_token,
"mask_token": mask_token,
"clean_text": clean_text,
"handle_chinese_chars": handle_chinese_chars,
"strip_accents": strip_accents,
"lowercase": lowercase,
"wordpieces_prefix": wordpieces_prefix,
}
super().__init__(tokenizer, parameters)
import tensorflow as tf
import numpy as np
from tokenizers import ByteLevelBPETokenizer as Tokenizer
from transformers import RobertaConfig as Config
import re
PATH = 'roberta-base'
MAX_SEQUENCE_LENGTH = 192
TOKENIZER = Tokenizer(vocab_file="roberta/vocab.json",
merges_file="roberta/merges.txt",
lowercase=True,
add_prefix_space=True)
def preprocess(tweet, selected_text, sentiment, training=True):
"""
Will be used in tf.data.Dataset.from_generator(...)
"""
# The original strings have been converted to
# byte strings, so we need to decode it
tweet = tweet.decode('utf-8')
selected_text = selected_text.decode('utf-8')
sentiment = sentiment.decode('utf-8')
# Clean up the strings a bit
tweet = " ".join(str(tweet).split())
selected_text = " ".join(str(selected_text).split())
import tensorflow as tf
import gpu_check
from preprocessing_data import create_training_data
from tokenizers import Tokenizer
from model_chatbot import seq2seq
from Hyper_parameter import (VOCAB_SIZE,
MAXLEN,
EPOCHS,
SAVE_AT,
LEARNING_RATE,
BATCH_SIZE,
VERBOSE,
LOSS)
tokenizer = Tokenizer()
encoder_input_data, decoder_input_data, decoder_output_data = create_training_data() # parsing the dataset and creating conversation pairs
encoder_input_data, decoder_input_data, decoder_output_data = tokenizer.tokenize_and_pad_training_data(encoder_input_data,
decoder_input_data,
decoder_output_data) # tokenizing and padding those pairs
tokenizer.save_tokenizer(f'tokenizer-vocab_size-{VOCAB_SIZE}') # saving tokenizer for layer use
Seq2SeqModel = seq2seq() # creating the seq2seq model
optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE, clipnorm=1.0, clipvalue=0.5)
Seq2SeqModel.compile(optimizer=optimizer, loss=LOSS, metrics=['accuracy'])
Seq2SeqModel.summary()
def train(model, encoder_input_data, decoder_input_data, decoder_output_data, epochs, batch_size, verbose, save_at):
def __init__(self):
self.tokenizer = Tokenizer(BPE())
self.tokenizer.normalizer = Sequence([NFKC()])
self.tokenizer.pre_tokenizer = ByteLevel()
self.tokenizer.decoder = ByteLevelDecoder()
def __init__(
self,
vocab_file: Optional[str] = None,
add_special_tokens: bool = True,
unk_token: str = "[UNK]",
sep_token: str = "[SEP]",
cls_token: str = "[CLS]",
clean_text: bool = True,
handle_chinese_chars: bool = True,
strip_accents: bool = True,
lowercase: bool = True,
wordpieces_prefix: str = "##",
):
if vocab_file is not None:
tokenizer = Tokenizer(WordPiece.from_files(vocab_file, unk_token=unk_token))
else:
tokenizer = Tokenizer(WordPiece.empty())
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(unk_token) is not None:
tokenizer.add_special_tokens([unk_token])
if tokenizer.token_to_id(sep_token) is not None:
tokenizer.add_special_tokens([sep_token])
if tokenizer.token_to_id(cls_token) is not None:
tokenizer.add_special_tokens([cls_token])
tokenizer.normalizer = BertNormalizer(
clean_text=clean_text,
handle_chinese_chars=handle_chinese_chars,
strip_accents=strip_accents,
lowercase=lowercase,
)
tokenizer.pre_tokenizer = BertPreTokenizer()
if add_special_tokens and vocab_file is not None:
sep_token_id = tokenizer.token_to_id(sep_token)
if sep_token_id is None:
raise TypeError("sep_token not found in the vocabulary")
cls_token_id = tokenizer.token_to_id(cls_token)
if cls_token_id is None:
raise TypeError("cls_token not found in the vocabulary")
tokenizer.post_processor = BertProcessing(
(sep_token, sep_token_id), (cls_token, cls_token_id)
)
tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix)
parameters = {
"model": "BertWordPiece",
"add_special_tokens": add_special_tokens,
"unk_token": unk_token,
"sep_token": sep_token,
"cls_token": cls_token,
"clean_text": clean_text,
"handle_chinese_chars": handle_chinese_chars,
"strip_accents": strip_accents,
"lowercase": lowercase,
"wordpieces_prefix": wordpieces_prefix,
}
super().__init__(tokenizer, parameters)
def test_full_strip(self):
tokenizer = Tokenizer(BPE.empty())
tokenizer.normalizer = Strip(left=True, right=True)
output = tokenizer.normalize(" hello ")
assert output == "hello"
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
)
bert_tokenizer.train_from_iterator(sentences, trainer=trainer)
if serialize_path:
bert_tokenizer.save(serialize_path)
return bert_tokenizer
ids = bert_tokenizer.encode(sentences[10]).ids
bert_tokenizer.decode(ids)
from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, decoders, trainers
tokenizer = Tokenizer(models.Unigram())
tokenizer.normalizer = normalizers.NFKC()
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
tokenizer.decoders = decoders.ByteLevel()
trainer = trainers.UnigramTrainer(
vocab_size=20000,
initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
special_tokens=["<PAD>", "<BOS>", "<EOS>"],
)
tokenizer.train_from_iterator(sentences, trainer=trainer)
tokenizer.encode(sentences[4]).ids
tokenizer.decode(tokenizer.encode(sentences[4]).ids)
tokenizer.save('bert_out/test2')
def test_can_make_sequences(self):
tokenizer = Tokenizer(BPE.empty())
tokenizer.normalizer = Sequence([Lowercase(), Strip()])
output = tokenizer.normalize(" HELLO ")
assert output == "hello"
def test_lowercase(self):
tokenizer = Tokenizer(BPE.empty())
tokenizer.normalizer = Lowercase()
output = tokenizer.normalize("HELLO")
assert output == "hello"
from tokenizers import Tokenizer
from tokenizers.models import BPE, WordLevel
from tokenizers.processors import BertProcessing, TemplateProcessing
from tokenizers import trainers
from transformers import BertForMaskedLM
from transformers import BertTokenizerFast
from transformers import BertConfig
import ipdb
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "2"
uid_task_id_sequence_path = 'data/feature_sequence/uid_task_id.txt'
paths = [str(x) for x in Path(".").glob('data/feature_sequence/*.txt')]
tokenizer = Tokenizer(WordLevel())
tokenizer.pre_tokenizer = Whitespace()
# trainer = trainers.BpeTrainer(
trainer = trainers.WordPieceTrainer(
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
tokenizer.train(trainer, [uid_task_id_sequence_path])
tokenizer.post_processor = TemplateProcessing(
single="[CLS] $A [SEP]",
pair="[CLS] $A [SEP] $B:1 [SEP]:1",
special_tokens=[
("[CLS]", tokenizer.token_to_id("[CLS]")),
("[SEP]", tokenizer.token_to_id("[SEP]")),
],
)
# tokenizer.save_model("tmp")
def train_tokenizer(
input_file: str,
vocab_file: str,
temporary: str,
subset_size: int = 512000000,
vocab_size: int = 8000,
limit_alphabet: int = 6000,
unk_token: str = '<unk>',
control_tokens: List[str] = []):
r"""Train **WordPiece** tokenizer and save trained subword vocabulary.
Note:
Since tokenizers_ reads whole file data in training, this function
could occur memory errors if `input_file` is too large. Under the
assumption that `input_file` is shuffled randomly, the subset of input
corpus will be used in training.
Caution:
The subset of input corpus is saved in `temporary` directory. Please be
careful not to delete the file while executing this function.
Arguments:
input_file (str): Input file path.
vocab_file (str): Output vocabulary file path.
temporary (str): Temporary directory where the subset of corpus would
be saved.
subset_size (int): The maximum number of lines in the subset.
vocab_size (int): The number of subwords in the vocabulary.
limit_alphabet (int): The maximum number of alphabets in vocabulary.
unk_tokens (str): Unknown token in the vocabulary.
control_tokens (list): Control tokens in the vocabulary.
.. _tokenizers: https://github.com/huggingface/tokenizers
"""
# Create **WordPiece** model and add normalizer and pre-tokenizer.
# BERT-specific normalizer and pre-tokenizer are used.
tokenizer = Tokenizer(models.WordPiece())
tokenizer.normalizer = BertNormalizer(strip_accents=False)
tokenizer.pre_tokenizer = BertPreTokenizer()
# Split the head of input corpus file and save in `temporary` directory.
subset_file = random_filename(temporary)
_split_subset_from_file(input_file, subset_file, subset_size)
# Train the model with splitted subset of corpus.
trainer = WordPieceTrainer(vocab_size=vocab_size,
min_frequency=2,
show_progress=True,
limit_alphabet=limit_alphabet,
special_tokens=[unk_token] + control_tokens,
continuing_subword_prefix='##')
tokenizer.train(trainer, [subset_file])
# Save trained subword vocabulary in `temporary` directory and rename to
# `vocab_file`.
tokenizer.model.save(temporary)
os.rename(os.path.join(temporary, 'vocab.txt'), vocab_file)
# Remove temporary subset corpus.
os.remove(subset_file)
"""
def pre_tokenize(self, sentence):
return None
def decode(self, tokens):
return None
def tokenize(sentence):
output = tokenizer.encode(sentence).tokens
print(f"`{sentence}` tokenized to {output}")
return output
# Create a Tokenizer using a BPE model
bpe = models.BPE.from_files(args.vocab, args.merges)
tokenizer = Tokenizer(models.BPE.from_files(args.vocab, args.merges))
# Test the good custom classes
good_custom = GoodCustom()
good_pretok = pre_tokenizers.PreTokenizer.custom(good_custom)
good_decoder = decoders.Decoder.custom(good_custom)
tokenizer.pre_tokenizer = good_pretok
tokenizer.decoder = good_decoder
print("Tokenization will work with good custom:")
encoding = tokenizer.encode("Hey friend!")
print(f"IDS: {encoding.ids}")
print(f"TOKENS: {encoding.tokens}")
print(f"OFFSETS: {encoding.offsets}")
decoded = tokenizer.decode(encoding.ids)
def test_instantiation_from_tokenizers(self):
bert_tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
PreTrainedTokenizerFast(tokenizer_object=bert_tokenizer)
def train_custom_tokenizer(dataset,
token_model,
tknzr_file,
vocab_size,
vocab=None,
pretrain_fast=False,
max_input_chars_per_word=None,
eos_token=None,
bos_token=None,
pad_token=None,
mask_token=None,
unk_token=None):
"""
Building a Tokenizer using HuggingFace library. The pipeline seems to be:
- Model : algorithm that tokenizes, it is a mandatory
component. There are only 4 models implemented
(BPE, Unigram, WordLevel, WordPiece)
- Normalizer : some preprocessing that could happen before, but
doesn't necessarily have to
- Pre-Tokenizer : splitting the input according to some rules
- Post-Processing : needing to add some tokens/input after (mostly seems
to be eos, bos tokens)
- Decoder : certain previous pipeline steps need to be reversed
for proper decoding
- Trainer : The corresponding training algorithm for the model
Note : Some pre-processing might need to happen beforehand in previous
functions (might be easier using pandas before)
Input
token_model (str) : algorithm to use for tokenization
dataset (class) : a python iterator that goes through the data
to be used for training
token_dir (str) : directory with tokenizers
vocab_size (int) : size of the vocabulary to use
tokenFilename (str) : filename of particular token we want to
train. Will overwrite previously save files.
vocab (list of str) : models other than BPE can use non-mandatory
vocab as input
max_input_chars_per_word : used for WordPiece
Output
tokenizer : huggingFace Tokenizer object, our fully
trainer tokenizer
"""
special_token_lst = [
pad_token, bos_token, eos_token, mask_token, unk_token
]
# NFKC
normalizer_lst = []
pre_tokenizer_lst = [Whitespace, ByteLevel]
decoder_lst = []
bos_idx = special_token_lst.index(bos_token)
eos_idx = special_token_lst.index(eos_token)
if token_model == 'BPE':
model = BPE(unk_token=unk_token)
Trainer = BpeTrainer
elif token_model == 'Unigram':
model = Unigram(vocab=vocab)
Trainer = UnigramTrainer
elif token_model == 'WordLevel':
model = WordLevel(unk_token=unk_token, vocab=vocab)
Trainer = WordLevelTrainer
elif token_model == 'WordPiece':
model = WordPiece(unk_token=unk_token,
vocab=vocab,
max_input_chars_per_word=max_input_chars_per_word)
Trainer = WordPieceTrainer
else:
error_msg = f'Error: token_model ({token_model}) not an algorithm in%s' \
% VALID_TOKENIZATIONS
raise SystemExit(error_msg)
# instantiation
tokenizer = Tokenizer(model)
# Select a tokenization trainer
if vocab_size is None:
trainer = Trainer(show_progress=True, special_tokens=special_token_lst)
else:
trainer = Trainer(vocab_size=vocab_size,
show_progress=True,
special_tokens=special_token_lst)
# Set the normalizer
tokenizer.normalizer = normalizers.Sequence(
[fcn() for fcn in normalizer_lst])
# Set the pre-tokenizer
tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
[fcn() for fcn in pre_tokenizer_lst])
# Set the post-processing
tokenizer.post_processor = processors.TemplateProcessing(
single=bos_token + " $A " + eos_token,
special_tokens=[(bos_token, bos_idx), (eos_token, eos_idx)],
# pair=bos_token+" $A "+eos_token" $B:1 "+eos_token+":1",
)
# Set the decoder
if ByteLevel in pre_tokenizer_lst:
tokenizer.decoder = decoders.ByteLevel()
if Metaspace in pre_tokenizer_lst:
tokenizer.decoder = decoders.Metaspace()
if token_model == 'WordPiece':
tokenizer.decoder = decoders.WordPiece()
# creating iterator
def batch_iterator():
for i in np.arange(0, len(dataset)):
yield dataset[i]
# train call
tokenizer.train_from_iterator(trainer=trainer,
iterator=batch_iterator(),
length=len(dataset))
if Path(tknzr_file).exists():
print(f"Warning : overwriting previously save tokenizer with\
same filename ( {tknzr_file} ).")
tokenizer.save(tknzr_file)
if pretrain_fast:
tokenizer = PreTrainedTokenizerFast(tokenizer_file=tknzr_file)
else:
tokenizer = PreTrainedTokenizer(tokenizer_file=tknzr_file)
tokenizer.pad_token = pad_token
tokenizer.mask_token = mask_token
return tokenizer
def test_instantiation_from_tokenizers_json_file(self):
bert_tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
with tempfile.TemporaryDirectory() as tmpdirname:
bert_tokenizer.save(os.path.join(tmpdirname, "tokenizer.json"))
PreTrainedTokenizerFast(
tokenizer_file=os.path.join(tmpdirname, "tokenizer.json"))
def __init__(
self,
vocab_file: Optional[str] = None,
unk_token: Union[str, AddedToken] = "<unk>",
sep_token: Union[str, AddedToken] = "<sep>",
cls_token: Union[str, AddedToken] = "<cls>",
pad_token: Union[str, AddedToken] = "<pad>",
mask_token: Union[str, AddedToken] = "<mask>",
lowercase: bool = False,
unicode_normalizer: Optional[str] = None,
):
if vocab_file is not None:
tokenizer = Tokenizer(WordLevel(vocab_file))
else:
tokenizer = Tokenizer(WordLevel())
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
if tokenizer.token_to_id(str(sep_token)) is not None:
tokenizer.add_special_tokens([str(sep_token)])
if tokenizer.token_to_id(str(cls_token)) is not None:
tokenizer.add_special_tokens([str(cls_token)])
if tokenizer.token_to_id(str(pad_token)) is not None:
tokenizer.add_special_tokens([str(pad_token)])
if tokenizer.token_to_id(str(mask_token)) is not None:
tokenizer.add_special_tokens([str(mask_token)])
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
if lowercase:
normalizers += [Lowercase()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
if vocab_file is not None:
sep_token_id = tokenizer.token_to_id(str(sep_token))
if sep_token_id is None:
raise TypeError("sep_token not found in the vocabulary")
cls_token_id = tokenizer.token_to_id(str(cls_token))
if cls_token_id is None:
raise TypeError("cls_token not found in the vocabulary")
tokenizer.post_processor = processors.BertProcessing(
(str(sep_token), sep_token_id), (str(cls_token), cls_token_id))
parameters = {
"model": "WordLevel",
"unk_token": unk_token,
"sep_token": sep_token,
"cls_token": cls_token,
"pad_token": pad_token,
"mask_token": mask_token,
"lowercase": lowercase,
"unicode_normalizer": unicode_normalizer,
}
super().__init__(tokenizer, parameters)
import json
data_path = Path('/workspace/poetry2021.gt/data/pan_tadeusz6')
dataset_path = data_path / 'dataset'
vocab_path = data_path / 'vocab.json'
tokenizer_tmp_path = data_path / 'tokenizer_tmp'
tokenizer_path = data_path / 'tokenizer'
text_tokenizer = TextTokenizer(dataset_path)
text_tokenizer.load_vocab(vocab_path)
vocab = text_tokenizer.vocab
vocab_count = len(vocab.keys())
vocab.update({'<|endoftext|>': vocab_count})
tokenizer_tmp = Tokenizer(WordLevel(text_tokenizer.vocab))
tokenizer_tmp.pre_tokenizer = CharDelimiterSplit(' ')
tokenizer_tmp.post_processor = BertProcessing(
("<|endoftext|>", tokenizer_tmp.token_to_id("<|endoftext|>")),
("<|endoftext|>", tokenizer_tmp.token_to_id("<|endoftext|>")),
)
tokenizer_tmp_path.mkdir(parents=True, exist_ok=True)
tokenizer_tmp.save(str(tokenizer_tmp_path / "tokenizer.json"))
# Re-create as GPT2 compatible tokenizer
class GPT2CompatibleTokenizer(PreTrainedTokenizerFast):
def save_vocabulary(self,
default="./",
type=str,
help="Path to the output directory, where the files will be saved")
parser.add_argument("--name",
default="bert-wordpiece",
type=str,
help="The name of the output vocab files")
args = parser.parse_args()
files = glob.glob(args.files)
if not files:
print(f"File does not exist: {args.files}")
exit(1)
# Initialize an empty tokenizer
tokenizer = Tokenizer(models.WordPiece.empty())
# Customize all the steps
tokenizer.normalizer = normalizers.BertNormalizer.new(
clean_text=True,
handle_chinese_chars=True,
strip_accents=True,
lowercase=True,
)
tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer.new()
tokenizer.decoder = decoders.WordPiece.new()
# And then train
trainer = trainers.WordPieceTrainer.new(
vocab_size=50000,
min_frequency=2,
from tokenizers import Tokenizer, normalizers
from tokenizers.models import WordLevel
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.trainers import WordLevelTrainer
from tokenizers.processors import TemplateProcessing
t = Tokenizer(WordLevel(unk_token="[UNK]"))
t.pre_tokenizer = Whitespace()
trainer = WordLevelTrainer(special_tokens=["[UNK]", "[PAD]", "[CLS]", "[SEP]"])
t.post_processor = TemplateProcessing(
single="[CLS] $A [SEP]",
# ,
# pair="[CLS] $A [SEP] $B:1 [SEP]:1",
special_tokens=[
("[CLS]", 2),
("[SEP]", 3),
])
files = ['tok-train-shuf-tgt.tsv']
t.train(files, trainer)
t.save("code_tokenizer.json")
In the face of ambiguity, refuse the temptation to guess.
There should be one-- and preferably only one --obvious way to do it.
Although that way may not be obvious at first unless you're Dutch.
Now is better than never.
Although never is often better than *right* now.
If the implementation is hard to explain, it's a bad idea.
If the implementation is easy to explain, it may be a good idea.
Namespaces are one honking great idea -- let's do more of those!
""".split("\n")
if args.type == "gpt2":
print("Running GPT-2 tokenizer")
tok_p = GPT2Tokenizer.from_pretrained(args.vocab, args.merges)
# Create a Tokenizer using BPE
tok_r = Tokenizer(BPE.from_files(args.vocab, args.merges))
# Use ByteLevel PreTokenizer
tok_r.pre_tokenizer = pre_tokenizers.ByteLevel.new(add_prefix_space=False)
# Use ByteLevel Decoder
tok_r.decoder = decoders.ByteLevel.new()
elif args.type == "bert":
print("Running Bert tokenizer")
tok_p = BertTokenizer.from_pretrained(args.vocab)
tok_r = Tokenizer(
WordPiece.from_files(args.vocab,
unk_token="[UNK]",
max_input_chars_per_word=100))
tok_r.normalizer = BertNormalizer.new(
clean_text=True,
handle_chinese_chars=True,
import pandas as pd
from tokenizers import Tokenizer
from tokenizers.models import BPE, WordPiece, Unigram
from tokenizers.normalizers import Lowercase
from tokenizers.pre_tokenizers import Whitespace, Digits, Sequence
from tokenizers.trainers import BpeTrainer, WordPieceTrainer, UnigramTrainer
TRAIN_DATA_PATH = 'data/data_fusion_train.parquet'
OUTPUT_PATH = 'data/tokenizers/'
# Prepare data
train = pd.read_parquet(TRAIN_DATA_PATH, columns=['item_name'])
item_names = train.item_name.drop_duplicates().tolist()
# WordPiece tokenizer
tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
tokenizer.pre_tokenizer = Sequence([Whitespace(), Digits()])
tokenizer.normalizer = Lowercase()
trainer = WordPieceTrainer(
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],
vocab_size=70000)
tokenizer.train_from_iterator(item_names, trainer)
tokenizer.save(os.path.join(OUTPUT_PATH, 'wordpiece_70k.json'))
# BPE tokenizer
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
tokenizer.pre_tokenizer = Sequence([Whitespace(), Digits()])
tokenizer.normalizer = Lowercase()
trainer = BpeTrainer(
g = Reader(arch).stream_data()
with open(fp, "w") as f:
for s in g:
f.write(s)
f.write("\n\n")
elif args.file_type == 'txt':
shutil.copyfile(str(arch), str(fp))
data_files = glob(str(out_path / "*.txt"))
data_files = random.sample(data_files, int(0.2 * len(data_files)))
assert len(data_files) > 0, 'No data files found'
# Initialize a tokenizer
tokenizer = Tokenizer(models.BPE())
# Customize pre-tokenization and decoding
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=True)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.post_processor = processors.ByteLevel(trim_offsets=True)
tokenizer.normalizer = NFKC()
# And then train
trainer = trainers.BpeTrainer(vocab_size=args.vocab_size, min_frequency=2, special_tokens=["<|endoftext|>", "<|padding|>"])
tokenizer.train(trainer, data_files)
# And Save it
tokenizer_path = out_path / "byte-level-bpe.tokenizer.json"
tokenizer.save(str(tokenizer_path), pretty=True)
for line in tqdm(fin):
dp = json.loads(line.strip())
for d in enumerate(dp):
if "value" in d:
if "," in d["value"]:
print('Not cleaned up')
# Extract value/types from trees and store in comma separated raw file (all_raw.json)
with open("output/all_new_trees.json") as fin, open("output/all_raw.json",
"w") as fout:
for i, line in enumerate(tqdm(fin)):
dp = json.loads(line)
token_list = []
for d in dp:
if "value" in d:
token_list.append(d["value"])
elif "type" in d:
token_list.append(d["type"])
raw = ",".join(token_list)
print(json.dumps(raw), file=fout)
# Train tokenizer on raw file
tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
tokenizer.pre_tokenizer = CharDelimiterSplit(delimiter=",")
trainer = WordPieceTrainer(special_tokens=["[UNK]", "[PAD]"])
tokenizer.train(["output/all_raw.json"], trainer)
tokenizer.save("output/tokenizer.json")
def main():
args = build_argparser().parse_args()
# load vocabulary file for model
vocab = load_vocab_file(args.vocab)
log.debug("Loaded vocab file from {}, get {} tokens".format(
args.vocab, len(vocab)))
# create tokenizer
tokenizer = Tokenizer(BPE(str(args.vocab), str(args.merges)))
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tokenizer.decoder = decoders.ByteLevel()
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
ie = IECore()
# read IR
model_xml = args.model
model_bin = model_xml.with_suffix(".bin")
log.info('Reading model {}'.format(args.model))
ie_net = ie.read_network(model=model_xml, weights=model_bin)
# check input and output names
if len(ie_net.input_info) != 1:
raise RuntimeError(
'The demo expects model with single input, while provided {}'.
format(len(ie_net.input_info)))
if len(ie_net.outputs) != 1:
raise RuntimeError(
'The demo expects model with single output, while provided {}'.
format(len(ie_net.outputs)))
input_names = next(iter(ie_net.input_info))
output_names = next(iter(ie_net.outputs))
# load model to the device
ie_net_exec = ie.load_network(network=ie_net, device_name=args.device)
log.info('The model {} is loaded to {}'.format(args.model, args.device))
if args.input:
def prompts():
for prompt in args.input:
log.info("Input prompt: {}".format(prompt))
yield prompt
else:
def prompts():
while True:
yield input('Type input prompt (empty string to exit):')
# loop on user's or prepared prompts
for prompt in prompts():
if not prompt.strip():
break
# encode input
tokens = tokenizer.encode_batch([prompt])[0].ids
input_ids = np.array([tokens], dtype=np.int32)
# maximum number of tokens that can be processed by network at once
max_length = ie_net.input_info[input_names].input_data.shape[1]
eos_token_id = len(vocab) - 1
cur_input_len = input_ids.shape[-1]
# maximum number of tokens that will be generated
max_sample_token_num = args.max_sample_token_num + cur_input_len
t0 = time.perf_counter()
t_count = 0
while True:
# pad the rest of the request
pad_len = max_length - cur_input_len
model_input = np.concatenate(
(input_ids, [[eos_token_id] * pad_len]), axis=-1)
# create numpy inputs for IE
inputs = {
input_names: model_input,
}
# infer by IE
t_start = time.perf_counter()
res = ie_net_exec.infer(inputs=inputs)
t_end = time.perf_counter()
t_count += 1
log.info(
"Sequence of length {} is processed with {:0.2f} requests/sec ({:0.2} sec per request)"
.format(max_length, 1 / (t_end - t_start), t_end - t_start))
outputs = res[output_names]
next_token_logits = outputs[:, cur_input_len - 1, :]
# pre-process distribution
next_token_scores = process_logits(input_ids, next_token_logits,
eos_token_id)
if args.top_k > 0:
next_token_scores = get_top_k_logits(next_token_scores,
args.top_k)
if args.top_p < 1.0:
next_token_scores = get_top_p_logits(next_token_scores,
args.top_p)
# get next token id
probs = softmax(next_token_scores)
next_tokens = np.random.choice(probs.shape[-1],
1,
p=probs[0],
replace=True)
# update info for the next step
input_ids = np.concatenate((input_ids, [next_tokens]), axis=-1)
cur_input_len = input_ids.shape[-1]
if stop_criteria(input_ids, min(max_length, max_sample_token_num),
eos_token_id):
break
t1 = time.perf_counter()
text = tokenizer.decode_batch(input_ids)[0]
log.info(
"{} requests of {} length were processed in {:0.2f}sec ({:0.2}sec per request)"
.format(t_count, max_length, t1 - t0, (t1 - t0) / t_count))
# print result
log.info("GENERATED SEQUENCE: {}".format(text))
from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
#from tokenizers.pre_tokenizers import Whitespace
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
trainer = BpeTrainer(
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
#tokenizer.pre_tokenizer = Whitespace()
files = ['./processed/processed_wiki_ko.txt']
tokenizer.train(files, trainer)
tokenizer.save("wiki_tokenizer.json")
def train_tokenizer_vocab(dataset, style='BPE', force_retrain=True):
"""
if force_retrain: overwrite the stored tokenizer from tokenizers dir (by retraining)
else: load the tokenizer if it exists
"""
assert dataset in VALID_DATASETS
assert style in VALID_TOKENIZATIONS
tpath_expected = default_tpath(dataset, style)
train = True
if not force_retrain and os.path.isfile(tpath_expected):
tokenizer = Tokenizer.from_file(tpath_expected)
train = False
else:
print('%s tokenizer file does not exist; training new tokenizer' %
tpath_expected)
if train:
# load data associated with one of the valid datasets (from /data/ directory)
datafiles = load_dataset(dataset)
# Steps for each algo (e.g. BPE):
# - init Tokenizer using algo
# - specify algo specific trainer
# - specify any pre-processing of text (will affect decoding)
# see: https://huggingface.co/docs/tokenizers/python/latest/components.html#decoders
# - different training calls if its the arxiv dataset or wikitext
# see https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/
if style == 'BPE':
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
trainer = BpeTrainer(
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
tokenizer.pre_tokenizer = ByteLevel()
if dataset == 'arxiv':
tokenizer.train_from_iterator(datafiles, trainer=trainer)
else:
tokenizer.train(datafiles, trainer=trainer)
tokenizer.decoder = decoders.ByteLevel()
else:
assert style == 'WordLevel'
tokenizer = Tokenizer(WordLevel(unk_token="[UNK]"))
trainer = WordLevelTrainer(
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
tokenizer.pre_tokenizer = Whitespace()
if dataset == 'arxiv':
tokenizer.train_from_iterator(datafiles, trainer=trainer)
else:
tokenizer.train(datafiles, trainer=trainer)
tokenizer.decoder = decoders.WordPiece(
) # WordPiece seems to work (adds back spaces)
# Save to tokenizers directory
tokenizer.save(tpath_expected)
# Generate vocab object based on tokenizer.decoder() method
# ... TODO implement the same vocabulary functionality, or ensure it is present in Tokenizer and then code it elsewhere...
# Features we need to match:
# from torchtext.legacy.vocab import Vocab as RetiredVocab
# ntokens = len(vocab.stoi) ---> ntokens = tokenizer.(...)
# data = [torch.tensor([vocab[token] for token in tokenizer(item)],
# dtype=torch.long) for item in raw_text_iter]
# tokenized_text_ints = torch.tensor([vocab[token] for token in tokenized_text], dtype=torch.long)
# running_context_string = ' '.join([vocab.itos[src[k]] for k in range(src.shape[0])])
# unk_index = vocab.unk_index
vocab = None
return tokenizer, vocab
from tokenizers import Tokenizer
from tokenizers.decoders import ByteLevel as ByteLevelDecoder
from tokenizers.models import BPE
from tokenizers.normalizers import Lowercase, NFKC, Sequence
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.trainers import BpeTrainer
path_data = "../../ml-datasets/wmt14/tokenizer/"
path_train_src = "../../ml-datasets/wmt14/train.en"
path_train_tgt = "../../ml-datasets/wmt14/train.de"
tokenizer = Tokenizer(BPE())
tokenizer.normalizer = Sequence([
NFKC(),
Lowercase()
])
tokenizer.pre_tokenizer = ByteLevel()
tokenizer.decoder = ByteLevelDecoder()
trainer = BpeTrainer(vocab_size=25000, show_progress=True, initial_alphabet=ByteLevel.alphabet(),
min_frequency=2, special_tokens=["<pad>", "<s>", "</s>", "<unk>", "<mask>", ])
tokenizer.train(trainer, [path_train_src, path_train_tgt])
print("Trained vocab size: {}".format(tokenizer.get_vocab_size()))
tokenizer.model.save(path_data)
from tokenizers import Tokenizer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.models import WordLevel
VOCAB_FILE = "data/tx1_vocab.txt"
with open(VOCAB_FILE, "r") as f:
words = list(set(f.read().strip().split("\n")))
vocab = {}
for i, word in enumerate(["<pad>", "<unk>"] + words):
vocab[word] = i
tokenizer = Tokenizer(WordLevel(vocab, unk_token="<unk>"))
tokenizer.enable_padding(pad_token="<pad>")
tokenizer.pre_tokenizer = Whitespace()
tokenizer.save("data/tokenizer-LakhNES-tx1.json")
