def __init__(
self,
vocab: Optional[Union[str, Dict[str, int]]] = None,
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int,
int]]]] = None,
unk_token: Union[str, AddedToken] = "<unk>",
replacement: str = "▁",
add_prefix_space: bool = True,
dropout: Optional[float] = None,
):
if vocab is not None and merges is not None:
tokenizer = Tokenizer(
BPE(vocab, merges, dropout=dropout, unk_token=unk_token))
else:
tokenizer = Tokenizer(BPE())
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
tokenizer.normalizer = NFKC()
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(
replacement=replacement, add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.Metaspace(
replacement=replacement, add_prefix_space=add_prefix_space)
parameters = {
"model": "SentencePieceBPE",
"unk_token": unk_token,
"replacement": replacement,
"add_prefix_space": add_prefix_space,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def test_encode_add_special_tokens(self, roberta_files):
with pytest.deprecated_call():
tokenizer = Tokenizer(
BPE(roberta_files["vocab"], roberta_files["merges"]))
tokenizer.add_special_tokens(["<s>", "</s>"])
tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True)
tokenizer.post_processor = RobertaProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
# Can encode with special tokens
output_with_specials = tokenizer.encode("My name is John",
add_special_tokens=True)
assert output_with_specials.tokens == [
"<s>", "ĠMy", "Ġname", "Ġis", "ĠJohn", "</s>"
]
# Can encode without special tokens
output_without_specials = tokenizer.encode("My name is John",
add_special_tokens=False)
assert output_without_specials.tokens == [
"ĠMy", "Ġname", "Ġis", "ĠJohn"
]
def __init__(self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
unk_token: str = "<unk>",
replacement: str = "▁",
add_prefix_space: bool = True,
dropout: Optional[float] = None):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE.from_files(vocab_file,
merges_file,
dropout=dropout,
unk_token=unk_token))
else:
tokenizer = Tokenizer(BPE.empty())
tokenizer.add_special_tokens([unk_token])
tokenizer.normalizer = NFKC.new()
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace.new(
replacement=replacement, add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.Metaspace.new(
replacement=replacement, add_prefix_space=add_prefix_space)
parameters = {
"model": "SentencePieceBPE",
"unk_token": unk_token,
"replacement": replacement,
"add_prefix_space": add_prefix_space,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def __init__(
self,
vocab_file: Optional[str] = None,
merges_file: Optional[str] = None,
unk_token: Union[str, AddedToken] = "<unk>",
replacement: str = "▁",
add_prefix_space: bool = True,
no_consecutive_space: bool = True,
dropout: Optional[float] = None,
clean_text: bool = True,
handle_chinese_chars: bool = True,
separate_numbers: bool = True,
strip_accents: bool = True,
lowercase: bool = True,
wordpieces_prefix: str = "##",
special_chars: str = SPECIAL_CHARS,
zh_norm: bool = True,
):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(
BPE(vocab_file,
merges_file,
dropout=dropout,
unk_token=unk_token))
else:
tokenizer = Tokenizer(BPE())
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
tokenizer.normalizer = Sequence([
NFKC(),
BertNormalizer(clean_text=clean_text,
handle_chinese_chars=handle_chinese_chars,
separate_numbers=separate_numbers,
strip_accents=strip_accents,
lowercase=lowercase,
special_chars=special_chars,
zh_norm=zh_norm)
])
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(
replacement=replacement,
add_prefix_space=add_prefix_space,
no_consecutive_space=no_consecutive_space)
tokenizer.decoder = decoders.Metaspace(
replacement=replacement,
add_prefix_space=add_prefix_space,
no_consecutive_space=no_consecutive_space)
parameters = {
"model": "SentencePieceBPE",
"unk_token": unk_token,
"replacement": replacement,
"add_prefix_space": add_prefix_space,
"no_consecutive_space": no_consecutive_space,
"dropout": dropout,
}
super().__init__(tokenizer, parameters)
def test_processing(self):
tokenizer = Tokenizer(BPE())
tokenizer.add_special_tokens(["<s>", "</s>"])
tokenizer.add_tokens(["my", "name", "is", "john", "pair"])
tokenizer.post_processor = RobertaProcessing(("</s>", 1), ("<s>", 0))
output = tokenizer.encode("my name", "pair")
assert output.tokens == ["<s>", "my", "name", "</s>", "</s>", "pair", "</s>"]
assert output.ids == [0, 2, 3, 1, 1, 6, 1]
def test_processing(self):
tokenizer = Tokenizer(BPE())
tokenizer.add_special_tokens(["[SEP]", "[CLS]"])
tokenizer.add_tokens(["my", "name", "is", "john", "pair"])
tokenizer.post_processor = BertProcessing(("[SEP]", 0), ("[CLS]", 1))
output = tokenizer.encode("my name", "pair")
assert output.tokens == ["[CLS]", "my", "name", "[SEP]", "pair", "[SEP]"]
assert output.ids == [1, 2, 3, 0, 6, 0]
def test_roberta_parity(self):
tokenizer = Tokenizer(BPE())
tokenizer.add_special_tokens(["<s>", "</s>"])
tokenizer.add_tokens(["my", "name", "is", "john", "pair"])
tokenizer.post_processor = RobertaProcessing(("</s>", 1), ("<s>", 0))
original = tokenizer.encode("my name is john", "pair")
tokenizer.post_processor = self.get_roberta()
template = tokenizer.encode("my name is john", "pair")
assert original.ids == template.ids
def test_add_special_tokens(self):
tokenizer = Tokenizer(BPE())
# Can add special tokens as `str`
added = tokenizer.add_special_tokens(["my", "name", "is", "john"])
assert added == 4
# Can add special tokens as `AddedToken`
added = tokenizer.add_special_tokens([AddedToken("the"), AddedToken("quick", rstrip=True)])
assert added == 2
def __init__(
self,
vocab_file: Optional[str] = None,
add_special_tokens: bool = True,
unk_token: str = "[UNK]",
sep_token: str = "[SEP]",
cls_token: str = "[CLS]",
clean_text: bool = True,
handle_chinese_chars: bool = True,
strip_accents: bool = True,
lowercase: bool = True,
wordpieces_prefix: str = "##",
):
if vocab_file is not None:
tokenizer = Tokenizer(
WordPiece.from_files(vocab_file, unk_token=unk_token))
else:
tokenizer = Tokenizer(WordPiece.empty())
tokenizer.add_special_tokens([unk_token, sep_token, cls_token])
tokenizer.normalizer = BertNormalizer(
clean_text=clean_text,
handle_chinese_chars=handle_chinese_chars,
strip_accents=strip_accents,
lowercase=lowercase,
)
tokenizer.pre_tokenizer = BertPreTokenizer()
if add_special_tokens and vocab_file is not None:
sep_token_id = tokenizer.token_to_id(sep_token)
if sep_token_id is None:
raise TypeError("sep_token not found in the vocabulary")
cls_token_id = tokenizer.token_to_id(cls_token)
if cls_token_id is None:
raise TypeError("cls_token not found in the vocabulary")
tokenizer.post_processor = BertProcessing(
(sep_token, sep_token_id), (cls_token, cls_token_id))
tokenizer.decoders = decoders.WordPiece(prefix=wordpieces_prefix)
parameters = {
"model": "BertWordPiece",
"add_special_tokens": add_special_tokens,
"unk_token": unk_token,
"sep_token": sep_token,
"cls_token": cls_token,
"clean_text": clean_text,
"handle_chinese_chars": handle_chinese_chars,
"strip_accents": strip_accents,
"lowercase": lowercase,
"wordpieces_prefix": wordpieces_prefix,
}
super().__init__(tokenizer, parameters)
def test_bert_parity(self):
tokenizer = Tokenizer(BPE())
tokenizer.add_special_tokens(["[SEP]", "[CLS]"])
tokenizer.add_tokens(["my", "name", "is", "john", "pair"])
tokenizer.post_processor = BertProcessing(("[SEP]", 0), ("[CLS]", 1))
original = tokenizer.encode("my name", "pair")
tokenizer.post_processor = self.get_bert()
template = tokenizer.encode("my name", "pair")
assert original.ids == template.ids
def load_or_train_tokenizer(file_paths, tokenizer_mode_path):
'''
Tries to load saved text tokenizer
If there is none, trains the new tokenizer and saves is
'''
if not os.path.exists(tokenizer_mode_path):
print('Tokenizer model not found, training one')
from tokenizers.models import BPE
from tokenizers import Tokenizer
from tokenizers.decoders import ByteLevel as ByteLevelDecoder
from tokenizers.normalizers import NFKC, Sequence
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.trainers import BpeTrainer
tokenizer = Tokenizer(BPE())
tokenizer.normalizer = Sequence([
NFKC()
])
tokenizer.pre_tokenizer = ByteLevel()
tokenizer.decoder = ByteLevelDecoder()
trainer = BpeTrainer(
vocab_size=50000,
show_progress=True,
inital_alphabet=ByteLevel.alphabet(),
special_tokens=[
"<s>",
"<pad>",
"</s>",
"<unk>",
"<mask>"
]
)
tokenizer.train(file_paths, trainer)
if not os.path.exists(tokenizer_mode_path):
os.makedirs(tokenizer_mode_path)
tokenizer.model.save(tokenizer_mode_path, None)
print('Loading trained tokenizer model')
tokenizer = GPT2Tokenizer.from_pretrained(tokenizer_mode_path)
tokenizer.add_special_tokens({
'eos_token': '</s>',
'bos_token': '<s>',
'unk_token': '<unk>',
'pad_token': '<pad>',
'mask_token': '<mask>'
})
return tokenizer
def tokenize_corpus(
input_file: str,
output_file: str,
vocab_file: str,
unk_token: str = '<unk>',
control_tokens: List[str] = []):
r"""Tokenize corpus sentences through trained **WordPiece** model.
Arguments:
input_file (str): Input corpus file path.
output_file (str): Output file path.
vocab_file (str): Trained vocabulary file path.
unk_token (str): Unknown token in the vocabulary.
control_tokens (list): Control tokens in the vocabulary.
"""
# Create `WordPiece` model and add special tokens. Note that `unk_token`
# is also a special token.normalizer and pre-tokenizer.
tokenizer = Tokenizer(models.WordPiece(vocab_file, unk_token=unk_token))
tokenizer.add_special_tokens([unk_token] + control_tokens)
# Use BERT-specific normalizer, pre-tokenizer and **WordPiece** decoder.
tokenizer.normalizer = BertNormalizer(strip_accents=False)
tokenizer.pre_tokenizer = BertPreTokenizer()
tokenizer.decoder = decoders.WordPiece(prefix='##')
with open(input_file, 'r', encoding='utf-8') as src, \
open(output_file, 'w', encoding='utf-8') as dst:
# Count total lines in corpus.
total_lines = 0
for _ in src:
total_lines += 1
# Move the corpus file to first.
src.seek(0)
buffer = []
for line in tqdm.tqdm(src,
desc='[*] tokenize corpus',
total=total_lines):
buffer.append(line)
# Tokenize buffered sentences and write to `output_file`.
if len(buffer) > 10000:
for t in tokenizer.encode_batch(buffer):
dst.write(' '.join(t.tokens) + '\n')
buffer.clear()
# Process the remained buffer.
if buffer:
for t in tokenizer.encode_batch(buffer):
dst.write(' '.join(t.tokens) + '\n')
def get_tokenizer(self, tokenizer_dir):
tokenizer = Tokenizer(models.BPE())
tokenizer.normalizer = Sequence(
[NFKC(), Replace('\r', ''),
Replace('\n', ' ')])
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
tokenizer.decoder = decoders.ByteLevel()
vocab_fn = os.path.join(tokenizer_dir, 'vocab.json')
merge_fn = os.path.join(tokenizer_dir, 'merges.txt')
tokenizer.model = models.BPE.from_file(vocab_fn, merge_fn)
tokenizer.add_special_tokens(['[UNK]', '[PAD]', '[BOS]', '[EOS]'])
return tokenizer
def test_add_special_tokens(self):
tokenizer = Tokenizer(BPE())
# Can add special tokens as `str`
added = tokenizer.add_special_tokens(["my", "name", "is", "john"])
assert added == 4
# Can add special tokens as `AddedToken`
tokens = [AddedToken("the"), AddedToken("quick", normalized=True), AddedToken()]
assert tokens[0].normalized == True
added = tokenizer.add_special_tokens(tokens)
assert added == 2
assert tokens[0].normalized == False
assert tokens[1].normalized == True
def __init__(
self,
vocab_file,
sep_token="<sep>",
cls_token="<cls>",
pad_token="<pad>",
mask_token="<mask>",
lowercase: bool = True,
):
tokenizer = Tokenizer(WordLevel(vocab_file, unk_token=unk_token))
tokenizer.normalizer = Strip()
tokenizer.pre_tokenizer = CharDelimiterSplit(" ")
tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
if tokenizer.token_to_id(str(sep_token)) is not None:
tokenizer.add_special_tokens([str(sep_token)])
if tokenizer.token_to_id(str(cls_token)) is not None:
tokenizer.add_special_tokens([str(cls_token)])
if tokenizer.token_to_id(str(pad_token)) is not None:
tokenizer.add_special_tokens([str(pad_token)])
if tokenizer.token_to_id(str(mask_token)) is not None:
tokenizer.add_special_tokens([str(mask_token)])
parameters = {
"model": "WordLevel",
"unk_token": unk_token,
"sep_token": sep_token,
"cls_token": cls_token,
"pad_token": pad_token,
"mask_token": mask_token,
"lowercase": lowercase,
}
super().__init__(tokenizer, parameters)
def build(self, afm: AuxiliaryFileManager, corpus: AuxiliaryFile,
vocab: AuxiliaryFile) -> AuxiliaryFile:
total_lines = self._total_lines_in_file(corpus)
# Create WordPiece model and add special tokens. Note that `unk_token`
# is also a special token.
tokenizer = Tokenizer(WordPiece(vocab.name, unk_token=self.unk_token))
tokenizer.add_special_tokens(self.special_tokens + [self.unk_token])
# Use BERT-specific normalizer, pre-tokenizer and decoder.
tokenizer.normalizer = BertNormalizer(strip_accents=False)
tokenizer.pre_tokenizer = BertPreTokenizer()
tokenizer.decoder = WordPieceDecoder(prefix='##')
tokenized = afm.create()
with corpus.open('r') as src, tokenized.open('w') as dst:
# Create tqdm progress bar with colorful description.
tqdm_iter = tqdm.tqdm(src,
desc=colorful.render(
'<r>[*]</r> tokenize sentences with '
'<g>WordPiece</g> model'),
total=total_lines)
batch_lines = []
for line in tqdm_iter:
batch_lines.append(line)
# Encode the grouped batch sentences and write the tokenized
# sentences to the auxiliary output file.
if len(batch_lines) > self.batch_size:
for t in tokenizer.encode_batch(batch_lines):
dst.write(' '.join(t.tokens) + '\n')
batch_lines.clear()
# Encode the remainders and write to the output file.
if batch_lines:
for t in tokenizer.encode_batch(batch_lines):
dst.write(' '.join(t.tokens) + '\n')
return tokenized
def converted(self) -> Tokenizer:
vocab = self.original_tokenizer.encoder
merges = list(self.original_tokenizer.bpe_ranks.keys())
unk_token = self.original_tokenizer.unk_token
tokenizer = Tokenizer(
BPE(
vocab=vocab,
merges=merges,
dropout=None,
unk_token=str(unk_token),
end_of_word_suffix="</w>",
fuse_unk=False,
))
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
tokenizer.normalizer = normalizers.BertNormalizer(lowercase=True)
tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()
tokenizer.decoder = decoders.BPEDecoder(suffix="</w>")
return tokenizer
def __init__(
self,
vocab_file: Optional[str] = None,
unk_token: Union[str, AddedToken] = "[UNK]",
pad_token: Union[str, AddedToken] = "[PAD]",
mask_token: Union[str, AddedToken] = "[MASK]",
lowercase: bool = False,
unicode_normalizer: Optional[str] = None,
):
if vocab_file is not None:
logging.info(f"Initiating tokenizer at {vocab_file}")
tokenizer = Tokenizer(
WordLevel(vocab=vocab_file, unk_token=unk_token))
else:
tokenizer = Tokenizer(WordLevel(unk_token=unk_token))
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
if tokenizer.token_to_id(str(pad_token)) is not None:
tokenizer.add_special_tokens([str(pad_token)])
if tokenizer.token_to_id(str(mask_token)) is not None:
tokenizer.add_special_tokens([str(mask_token)])
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
if lowercase:
normalizers += [Lowercase()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
parameters = {
"model": "WordLevel",
"unk_token": unk_token,
"pad_token": pad_token,
"mask_token": mask_token,
"lowercase": lowercase,
"unicode_normalizer": unicode_normalizer,
}
super().__init__(tokenizer, parameters)
def preprocess_data(args):
label_counter = Counter([])
examples_per_file = Counter()
print("Reading all files for labels.")
for input_file in args.input_files:
with xopen(input_file, "rt") as f:
for example, labels in input_readers[args.task](f):
examples_per_file[input_file] += 1
label_counter.update(labels)
if args.top_n_labels > 0:
mlb_full = MultiLabelBinarizer(sparse_output=True)
mlb_full = mlb_full.fit(label_counter.keys())
label_counter = dict(label_counter.most_common(args.top_n_labels))
mlb = MultiLabelBinarizer(sparse_output=True)
# Passing a list in a list because that's what the function wants.
mlb = mlb.fit([[pair for pair in label_counter]])
# Save list of partial -> full mapping if doing top N labels.
if args.top_n_labels > 0:
label_mapping = np.where(np.in1d(mlb_full.classes_,
mlb.classes_))[0].tolist()
with xopen(args.label_mapping, "wt") as f:
f.write(json.dumps(label_mapping))
# Also save the full labels.
with xopen(args.full_labels, "wt") as f:
f.write(json.dumps(list(mlb_full.classes_)))
# Save list of labels.
with xopen(args.labels_out, "wt") as f:
f.write(json.dumps(list(mlb.classes_)))
# Set parallel tokenization thread count.
os.environ["RAYON_NUM_THREADS"] = str(args.processes)
from tokenizers import Tokenizer, decoders, trainers
from tokenizers.models import WordPiece
from tokenizers.normalizers import BertNormalizer
from tokenizers.pre_tokenizers import BertPreTokenizer
from tokenizers.processors import BertProcessing
if args.task == 'cafa':
# Define our custom tokenizer.
# It is exactly the same as the default BERT tokenizer, except for max_input_chars_per_word
# being 20000 instead of 100. This tokenizer is very slow on the long protein sequences.
tokenizer = WordPiece.from_files(args.vocab,
unk_token="[UNK]",
max_input_chars_per_word=20000)
tokenizer = Tokenizer(tokenizer)
tokenizer.add_special_tokens(["[UNK]", "[SEP]", "[CLS]"])
tokenizer.normalizer = BertNormalizer(lowercase=args.do_lower_case)
tokenizer.pre_tokenizer = BertPreTokenizer()
tokenizer.post_processor = BertProcessing(
("[SEP]", tokenizer.token_to_id("[SEP]")),
("[CLS]", tokenizer.token_to_id("[CLS]")))
tokenizer.decoder = decoders.WordPiece(prefix='##')
else:
tokenizer = BertWordPieceTokenizer(args.vocab,
lowercase=args.do_lower_case)
tokenizer.enable_padding(max_length=args.seq_len)
tokenizer.enable_truncation(max_length=args.seq_len)
for input_file in args.input_files:
with xopen(input_file, 'rt') as in_f:
file_name = generate_out_filename(input_file, args)
with xopen(file_name, "wt") as out_f:
print("Processing to: ", file_name)
# Write the shape as the first row, useful for the finetuning.
out_f.write(
json.dumps((examples_per_file[input_file],
len(label_counter))) + '\n')
batch_size = min(examples_per_file[input_file],
args.processes * 100)
example_batch = []
labels_batch = []
with ParallelGenerator(input_readers[args.task](in_f),
max_lookahead=batch_size) as g:
for example, labels in g:
example_batch.append(example)
labels_batch.append(labels)
if len(example_batch) == batch_size:
example_batch = tokenizer.encode_batch(
example_batch)
labels_batch = mlb.transform(labels_batch)
for example, labels in zip(example_batch,
labels_batch):
# Convert sparse arrays to python lists for json dumping.
# print(labels);input()
labels = labels.nonzero()[1].tolist()
out_f.write(
json.dumps([example.ids, labels]) + '\n')
example_batch = []
labels_batch = []
# Write out whatever is left in the last smaller batch.
example_batch = tokenizer.encode_batch(example_batch)
labels_batch = mlb.transform(labels_batch)
for example, labels in zip(example_batch, labels_batch):
# Convert sparse arrays to python lists for json dumping.
# print(labels);input()
labels = labels.nonzero()[1].tolist()
out_f.write(json.dumps([example.ids, labels]) + '\n')
def __init__(
self,
vocab: Optional[Union[str, Dict[str, int]]] = None,
unk_token: Union[str, AddedToken] = "[UNK]",
sep_token: Union[str, AddedToken] = "[SEP]",
cls_token: Union[str, AddedToken] = "[CLS]",
pad_token: Union[str, AddedToken] = "[PAD]",
mask_token: Union[str, AddedToken] = "[MASK]",
clean_text: bool = True,
handle_chinese_chars: bool = True,
strip_accents: Optional[bool] = None,
lowercase: bool = True,
wordpieces_prefix: str = "##",
):
if vocab is not None:
tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(unk_token)))
else:
tokenizer = Tokenizer(WordPiece(unk_token=str(unk_token)))
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
if tokenizer.token_to_id(str(sep_token)) is not None:
tokenizer.add_special_tokens([str(sep_token)])
if tokenizer.token_to_id(str(cls_token)) is not None:
tokenizer.add_special_tokens([str(cls_token)])
if tokenizer.token_to_id(str(pad_token)) is not None:
tokenizer.add_special_tokens([str(pad_token)])
if tokenizer.token_to_id(str(mask_token)) is not None:
tokenizer.add_special_tokens([str(mask_token)])
tokenizer.normalizer = BertNormalizer(
clean_text=clean_text,
handle_chinese_chars=handle_chinese_chars,
strip_accents=strip_accents,
lowercase=lowercase,
)
tokenizer.pre_tokenizer = BertPreTokenizer()
if vocab is not None:
sep_token_id = tokenizer.token_to_id(str(sep_token))
if sep_token_id is None:
raise TypeError("sep_token not found in the vocabulary")
cls_token_id = tokenizer.token_to_id(str(cls_token))
if cls_token_id is None:
raise TypeError("cls_token not found in the vocabulary")
tokenizer.post_processor = BertProcessing(
(str(sep_token), sep_token_id), (str(cls_token), cls_token_id))
tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix)
parameters = {
"model": "BertWordPiece",
"unk_token": unk_token,
"sep_token": sep_token,
"cls_token": cls_token,
"pad_token": pad_token,
"mask_token": mask_token,
"clean_text": clean_text,
"handle_chinese_chars": handle_chinese_chars,
"strip_accents": strip_accents,
"lowercase": lowercase,
"wordpieces_prefix": wordpieces_prefix,
}
super().__init__(tokenizer, parameters)
def preprocess_data(args):
label_counter = Counter([])
examples_per_file = Counter()
print("Reading all files for labels.")
for input_file in args.input_files:
with xopen(input_file, "rt") as f:
for example, labels in input_readers[args.task](f):
examples_per_file[input_file] += 1
label_counter.update(labels)
if args.top_n_labels > 0:
mlb_full = MultiLabelBinarizer(sparse_output=True)
mlb_full = mlb_full.fit(label_counter.keys())
label_counter = dict(label_counter.most_common(args.top_n_labels))
mlb = MultiLabelBinarizer(sparse_output=True)
# Passing a list in a list because that's what the function wants.
if args.labels_in:
labels = json.load(open(args.labels_in))
mlb = mlb.fit([labels])
else:
mlb = mlb.fit([[pair for pair in label_counter]])
# Save list of partial -> full mapping if doing top N labels.
if args.top_n_labels > 0:
label_mapping = np.where(np.in1d(mlb_full.classes_,
mlb.classes_))[0].tolist()
with xopen(args.label_mapping, "wt") as f:
f.write(json.dumps(label_mapping))
# Also save the full labels.
with xopen(args.full_labels, "wt") as f:
f.write(json.dumps(list(mlb_full.classes_)))
# Save list of labels.
with xopen(args.labels_out, "wt") as f:
f.write(json.dumps(list(mlb.classes_)))
# Set parallel tokenization thread count.
os.environ["RAYON_NUM_THREADS"] = str(args.processes)
from tokenizers import Tokenizer, decoders, trainers
from tokenizers.models import WordPiece
from tokenizers.normalizers import BertNormalizer
from tokenizers.pre_tokenizers import BertPreTokenizer
from tokenizers.processors import BertProcessing
if args.task == 'cafa':
# Define our custom tokenizer.
# It is exactly the same as the default BERT tokenizer, except for max_input_chars_per_word
# being 20000 instead of 100. This tokenizer is very slow on the long protein sequences.
tokenizer = WordPiece.from_files(args.vocab,
unk_token="[UNK]",
max_input_chars_per_word=20000)
tokenizer = Tokenizer(tokenizer)
tokenizer.add_special_tokens(["[UNK]", "[SEP]", "[CLS]"])
tokenizer.normalizer = BertNormalizer(lowercase=args.do_lower_case)
tokenizer.pre_tokenizer = BertPreTokenizer()
tokenizer.post_processor = BertProcessing(
("[SEP]", tokenizer.token_to_id("[SEP]")),
("[CLS]", tokenizer.token_to_id("[CLS]")))
tokenizer.decoder = decoders.WordPiece(prefix='##')
else:
tokenizer = BertWordPieceTokenizer(args.vocab,
lowercase=args.do_lower_case)
tokenizer.enable_padding(max_length=args.seq_len)
tokenizer.enable_truncation(max_length=args.seq_len)
for input_file in args.input_files:
with xopen(input_file, 'rt') as in_f:
file_name = generate_out_filename(input_file, args)
with xopen(file_name, "wt") as out_f:
print("Processing to: ", file_name)
# Write the shape as the first row, useful for the finetuning.
if args.labels_in:
n_labels = len(json.load(open(args.labels_in)))
else:
n_labels = len(label_counter)
out_f.write(
json.dumps((examples_per_file[input_file], n_labels)) +
'\n')
batch_size = min(examples_per_file[input_file],
args.processes * 100)
example_batch = []
labels_batch = []
doc_idx_batch = []
with ParallelGenerator(input_readers[args.task](in_f),
max_lookahead=batch_size) as g:
START_POS = int(args.window_start) / 100
for doc_idx, (example, labels) in enumerate(g):
#example = ' '.join(example.split(' ')[-510:])
example_batch.append(example)
labels_batch.append(labels)
doc_idx_batch.append(doc_idx)
if len(example_batch) == batch_size:
example_batch = tokenizer.encode_batch(
example_batch)
labels_batch = mlb.transform(labels_batch)
for example, labels, doc_idx in zip(
example_batch, labels_batch,
doc_idx_batch):
# Convert sparse arrays to python lists for json dumping.
# print(labels);input()
labels = labels.nonzero()[1].tolist()
"""try:
[][0]
print("DOC_LEN:",len(example.overflowing)+1)
mid = len(example.overflowing)//2
out_f.write(json.dumps( [example.overflowing[mid].ids, labels, len(example.overflowing)+1] ) + '\n')
except IndexError:
out_f.write(json.dumps( [example.ids, labels, len(example.overflowing)+1] ) + '\n')"""
if args.all_blocks or args.n_blocks > 0:
blocks = [example.ids] + [
blk.ids for blk in example.overflowing
]
#print("BLOCKS:%d,TOKENS:%d" % (len(list(blocks)), sum([len(list(tokens)) for tokens in blocks])))
for b, block in enumerate(blocks, 2):
if b > args.n_blocks and args.n_blocks > 0:
break
out_f.write(
json.dumps(
[block, labels, doc_idx]) +
'\n')
else:
window = get_window(example, START_POS)
assert len(window) == 512
assert all(
[type(y) is int for y in window])
out_f.write(
json.dumps([window, labels]) + '\n')
example_batch = []
labels_batch = []
# Write out whatever is left in the last smaller batch.
example_batch = tokenizer.encode_batch(example_batch)
labels_batch = mlb.transform(labels_batch)
for example, labels, doc_idx in zip(
example_batch, labels_batch, doc_idx_batch):
# Convert sparse arrays to python lists for json dumping.
# print(labels);input()
labels = labels.nonzero()[1].tolist()
"""try:
[][0]
print("DOC_LEN:",len(example.overflowing)+1)
mid = len(example.overflowing)//2
out_f.write(json.dumps( [example.overflowing[mid].ids, labels, len(example.overflowing)+1] ) + '\n')
except IndexError:
out_f.write(json.dumps( [example.ids, labels, len(example.overflowing)+1] ) + '\n')"""
if args.all_blocks or args.n_blocks > 0:
blocks = [example.ids] + [
blk.ids for blk in example.overflowing
]
#print("BLOCKS:%d,TOKENS:%d" % (len(list(blocks)), sum([len(list(tokens)) for tokens in blocks])))
for b, block in enumerate(blocks, 2):
if b > args.n_blocks and args.n_blocks > 0:
break
out_f.write(
json.dumps([block, labels, doc_idx]) +
'\n')
else:
out_f.write(
json.dumps(
[get_window(example, START_POS), labels]) +
'\n')
class SentencePieceBPETokenizer:
"""Custom SentencePiece tokenizer"""
unk_token = '<unk>'
pad_token = '<pad>'
def __init__(self,
vocab: Dict[str, int] = None,
merges: List[Tuple[str, str]] = None,
dropout: float = None,
max_length: Optional[int] = 64) -> None:
"""Constructor
Args:
vocab (Dict[str, int]): A dictionary of string keys and their ids.
merges (List[Tuple[str, str]]): A list of pairs of tokens.
dropout (float): BPE dropout
max_length (int, optional): The max length at which to truncate.
Defaults to `64`.
"""
self.tokenizer = Tokenizer(
BPE(vocab, merges, dropout=dropout, unk_token=self.unk_token))
self.tokenizer.normalizer = BertNormalizer() # noqa
self.tokenizer.pre_tokenizer = pre_tokenizers.Metaspace() # noqa
self.tokenizer.decoder = decoders.Metaspace() # noqa
self.tokenizer.add_special_tokens([self.pad_token, self.unk_token])
self.tokenizer.enable_padding(pad_token=self.pad_token)
self.tokenizer.enable_truncation(max_length)
@classmethod
def train(cls,
dataset: Sequence[str],
vocab_size: int = 1000,
min_frequency: int = 2,
dropout: float = 0.0,
max_length: Optional[int] = 64) -> 'SentencePieceBPETokenizer':
instance = cls(dropout=dropout, max_length=max_length)
trainer = trainers.BpeTrainer(
vocab_size=vocab_size,
min_frequency=min_frequency,
special_tokens=[cls.pad_token, cls.unk_token])
instance.tokenizer.train_from_iterator(dataset, trainer=trainer)
instance.tokenizer.model.dropout = None
return instance
@property
def vocab_size(self):
return len(self.tokenizer.get_vocab())
def serialize(self):
return self.tokenizer.to_str()
@classmethod
def deserialize(cls, s: str) -> 'SentencePieceBPETokenizer':
tokenizer = cls()
tokenizer.tokenizer = Tokenizer.from_str(s)
return tokenizer
def encode(self, text: str) -> Dict[str, Any]:
encoding = self.tokenizer.encode(text)
outputs = {
'ids': torch.tensor(encoding.ids),
'mask': torch.tensor(encoding.attention_mask),
'spans': encoding.offsets,
}
return outputs
def encode_batch(self, batch: List[str]):
encodings = self.tokenizer.encode_batch(batch)
outputs = {
'ids': torch.tensor([e.ids for e in encodings]),
'mask': torch.tensor([e.attention_mask for e in encodings]),
'spans': [e.offsets for e in encodings],
}
return outputs
def __init__(
self,
target_vocab,
):
special_tokens = {
"pad_token": "[PAD]",
"unk_token": "[UNK]",
"sep_token": "[SEP]",
"cls_token": "[CLS]",
"mask_token": "[MASK]",
}
vocab = {}
vocab[special_tokens["pad_token"]] = 0
tkn_idx = 1
unused_ctr = 0
# not sure whether that's relevant, but fill 1..99 and 105...999
# with unused tokens to keep BERT's tokenizer style
# as a result, one can easily identify special tokens:
# 0 is padding
# 1xx are other special tokens
# any four-digit tokens are actual payload
fill_tokens = False
if(fill_tokens):
while(tkn_idx < 100):
vocab[f"[unused{unused_ctr}]"] = tkn_idx
tkn_idx += 1
unused_ctr += 1
for token in ["unk_token", "cls_token", "sep_token", "mask_token"]:
vocab[special_tokens[token]] = tkn_idx
tkn_idx += 1
if(fill_tokens):
while(tkn_idx < 1000):
vocab[f"[unused{unused_ctr}]"] = tkn_idx
tkn_idx += 1
unused_ctr += 1
for word in target_vocab:
vocab[word] = tkn_idx
tkn_idx += 1
tokenizer = Tokenizer(WordLevel(vocab=vocab, unk_token=special_tokens["unk_token"]))
tokenizer.add_special_tokens(list(special_tokens.values()))
tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
sep_token_id = tokenizer.token_to_id(special_tokens["sep_token"])
cls_token_id = tokenizer.token_to_id(special_tokens["cls_token"])
tokenizer.post_processor = processors.BertProcessing(
(special_tokens["sep_token"], sep_token_id), (special_tokens["cls_token"], cls_token_id)
)
parameters = special_tokens
parameters["model"] = "WordLevel"
super().__init__(tokenizer, parameters)
tokenizer.save(PRETRAINED_TOKENIZER_FILE)
def __init__(self,
vocab_file: Optional[str] = None,
unk_token: Union[str, AddedToken] = "<unk>",
sep_token: Union[str, AddedToken] = "</s>",
cls_token: Union[str, AddedToken] = "<s>",
nl_token: Union[str, AddedToken] = "<nl>",
pad_token: Union[str, AddedToken] = "<pad>",
mask_token: Union[str, AddedToken] = "<mask>",
clean_text: bool = True,
handle_chinese_chars: bool = True,
separate_numbers: bool = True,
strip_accents: bool = True,
lowercase: bool = True,
wordpieces_prefix: str = "##",
special_chars: str = SPECIAL_CHARS,
zh_norm: bool = True,
handle_simpl: bool = True,
do_postprocess: bool = False):
if vocab_file is not None:
tokenizer = Tokenizer(
WordPiece(vocab_file, unk_token=str(unk_token)))
else:
tokenizer = Tokenizer(WordPiece())
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
if tokenizer.token_to_id(str(sep_token)) is not None:
tokenizer.add_special_tokens([str(sep_token)])
if tokenizer.token_to_id(str(cls_token)) is not None:
tokenizer.add_special_tokens([str(cls_token)])
if tokenizer.token_to_id(str(pad_token)) is not None:
tokenizer.add_special_tokens([str(pad_token)])
if tokenizer.token_to_id(str(nl_token)) is not None:
tokenizer.add_special_tokens([str(nl_token)])
if tokenizer.token_to_id(str(mask_token)) is not None:
tokenizer.add_special_tokens([str(mask_token)])
if tokenizer.token_to_id(str(mask_token)) is not None:
tokenizer.add_special_tokens([str(mask_token)])
tokenizer.normalizer = Sequence([
NFKC(),
BertNormalizer(clean_text=clean_text,
handle_chinese_chars=handle_chinese_chars,
separate_numbers=separate_numbers,
strip_accents=strip_accents,
lowercase=lowercase,
special_chars=special_chars,
zh_norm=zh_norm,
handle_simpl=handle_simpl)
])
tokenizer.pre_tokenizer = BertPreTokenizer()
if vocab_file is not None and do_postprocess:
sep_token_id = tokenizer.token_to_id(str(sep_token))
if sep_token_id is None:
raise TypeError("sep_token not found in the vocabulary")
cls_token_id = tokenizer.token_to_id(str(cls_token))
if cls_token_id is None:
raise TypeError("cls_token not found in the vocabulary")
tokenizer.post_processor = BertProcessing(
(str(sep_token), sep_token_id), (str(cls_token), cls_token_id))
tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix)
parameters = {
"model": "BertWordPiece",
"unk_token": unk_token,
"sep_token": sep_token,
"cls_token": cls_token,
"nl_token": nl_token,
"pad_token": pad_token,
"mask_token": mask_token,
"clean_text": clean_text,
"handle_chinese_chars": handle_chinese_chars,
"separate_numbers": separate_numbers,
"strip_accents": strip_accents,
"lowercase": lowercase,
"special_chars": special_chars,
"zh_norm": zh_norm,
"handle_simpl": handle_simpl,
"wordpieces_prefix": wordpieces_prefix,
}
super().__init__(tokenizer, parameters)
class LitTokenizer:
def __init__(self,
padding=False,
truncation=False,
max_length=None,
lower=False,
lang=None):
super().__init__()
self.UNK_WORD = '[UNK]'
self.PAD_WORD = '[PAD]'
self.MASK_WORD = '[MASK]'
self.SOS_WORD = '[SOS]'
self.EOS_WORD = '[EOS]'
self.special_tokens = [
self.UNK_WORD, self.PAD_WORD, self.MASK_WORD, self.SOS_WORD,
self.EOS_WORD
]
# Define tokenizer
self.tokenizer = None
self.configure_tokenizers(padding, truncation, max_length, lower)
# Other
self.lang = lang
def get_vocab_size(self):
return self.tokenizer.get_vocab_size()
def configure_tokenizers(self, padding, truncation, max_length, lower):
# Settings
pad_length = None
if padding in {True, "longest"}:
pass
elif padding in {"max_length"}:
pad_length = max_length
elif padding in {False, "do_not_pad"}:
pass
else:
raise ValueError("Unknown padding type")
# SRC tokenizer
tok_normalizers = [NFD(), Strip()]
if lower:
tok_normalizers += [Lowercase()]
self.tokenizer = Tokenizer(tok_model()) # unk_token=... not working
self.tokenizer.add_special_tokens(self.special_tokens)
self.tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
[WhitespaceSplit()])
self.tokenizer.normalizer = normalizers.Sequence(
tok_normalizers) # StripAccents requires NFD
self.tokenizer.decoder = tok_decoder()
# Define template (Needed for the sos/eos tokens)
basic_template = TemplateProcessing(
single=f"{self.SOS_WORD} $A {self.EOS_WORD}",
pair=
f"{self.SOS_WORD} $A {self.EOS_WORD} {self.SOS_WORD} $B {self.EOS_WORD}",
special_tokens=[
(self.SOS_WORD, self.tokenizer.token_to_id(self.SOS_WORD)),
(self.EOS_WORD, self.tokenizer.token_to_id(self.EOS_WORD))
],
)
self.tokenizer.post_processor = basic_template
if padding:
self.tokenizer.enable_padding(pad_id=self.tokenizer.token_to_id(
self.PAD_WORD),
pad_token=self.PAD_WORD,
length=pad_length)
if truncation:
self.tokenizer.enable_truncation(max_length,
stride=0,
strategy='longest_first')
def load_vocab(self, vocab, merges):
vocab, merges = tok_model.read_file(vocab, merges)
self.tokenizer.model = tok_model(vocab, merges)
def train_vocab(self, files, vocab_size=32000, min_frequency=3):
# Train trainer
trainer = tok_trainer(vocab_size=vocab_size,
min_frequency=min_frequency)
self.tokenizer.train(files, trainer)
def save_vocab(self, output_dir, prefix):
self.tokenizer.model.save(output_dir, prefix)
def pad(self, examples, keys=None):
pad_idx = self.special_tokens.index(self.PAD_WORD)
# Keys to modify
if not keys:
keys = list(examples[0].keys())
d = {}
for k in keys:
# Collect same-type items (list of IDs, list of masks,...)
d[k] = [x[k] for x in examples]
# Get max length (value to pad)
max_length = max([x.shape[-1] for x in d[k]])
# Apply padding
for i, x in enumerate(examples):
unpadded_t = x[k]
if k == "ids":
tmp = torch.full((max_length, ),
fill_value=pad_idx,
device=unpadded_t.device) # All padding
elif k == "attention_mask":
tmp = torch.full(
(max_length, ), fill_value=0,
device=unpadded_t.device) # No attention mask
else:
raise TypeError("Unknown key")
tmp[:unpadded_t.shape[-1]] = unpadded_t
d[k][i] = tmp
return d
def encode(self, x):
return self.tokenizer.encode(x)
def decode(self, x):
if isinstance(x, torch.Tensor):
assert len(x.shape) == 2
x = x.detach().cpu().numpy()
return [self.tokenizer.decode(x_i) for x_i in x]
def __init__(
self,
vocab_file: Optional[str] = None,
unk_token: Union[str, AddedToken] = "<unk>",
sep_token: Union[str, AddedToken] = "<sep>",
cls_token: Union[str, AddedToken] = "<cls>",
pad_token: Union[str, AddedToken] = "<pad>",
mask_token: Union[str, AddedToken] = "<mask>",
lowercase: bool = False,
unicode_normalizer: Optional[str] = None,
):
if vocab_file is not None:
tokenizer = Tokenizer(WordLevel(vocab_file))
else:
tokenizer = Tokenizer(WordLevel())
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
if tokenizer.token_to_id(str(sep_token)) is not None:
tokenizer.add_special_tokens([str(sep_token)])
if tokenizer.token_to_id(str(cls_token)) is not None:
tokenizer.add_special_tokens([str(cls_token)])
if tokenizer.token_to_id(str(pad_token)) is not None:
tokenizer.add_special_tokens([str(pad_token)])
if tokenizer.token_to_id(str(mask_token)) is not None:
tokenizer.add_special_tokens([str(mask_token)])
# Check for Unicode normalization first (before everything else)
normalizers = []
if unicode_normalizer:
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
if lowercase:
normalizers += [Lowercase()]
# Create the normalizer structure
if len(normalizers) > 0:
if len(normalizers) > 1:
tokenizer.normalizer = Sequence(normalizers)
else:
tokenizer.normalizer = normalizers[0]
tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
if vocab_file is not None:
sep_token_id = tokenizer.token_to_id(str(sep_token))
if sep_token_id is None:
raise TypeError("sep_token not found in the vocabulary")
cls_token_id = tokenizer.token_to_id(str(cls_token))
if cls_token_id is None:
raise TypeError("cls_token not found in the vocabulary")
tokenizer.post_processor = processors.BertProcessing(
(str(sep_token), sep_token_id), (str(cls_token), cls_token_id))
parameters = {
"model": "WordLevel",
"unk_token": unk_token,
"sep_token": sep_token,
"cls_token": cls_token,
"pad_token": pad_token,
"mask_token": mask_token,
"lowercase": lowercase,
"unicode_normalizer": unicode_normalizer,
}
super().__init__(tokenizer, parameters)