class BertNerPreprocessor(Component):
"""Takes tokens and splits them into bert subtokens, encode subtokens with their indices.
Creates mask of subtokens (one for first subtoken, zero for later subtokens).
If tags are provided, calculate tags for subtokens.
Args:
vocab_file: path to vocabulary
do_lower_case: set True if lowercasing is needed
max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
max_subword_length: replace token to <unk> if it's length is larger than this
(defaults to None, which is equal to +infinity)
token_mask_prob: probability of masking token while training
provide_subword_tags: output tags for subwords or for words
Attributes:
max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
max_subword_length: rmax lenght of a bert subtoken
tokenizer: instance of Bert FullTokenizer
"""
def __init__(self,
vocab_file: str,
do_lower_case: bool = True,
max_seq_length: int = 512,
max_subword_length: int = None,
token_maksing_prob: float = 0.0,
provide_subword_tags: bool = False,
**kwargs):
self._re_tokenizer = re.compile(r"[\w']+|[^\w ]")
self.provide_subword_tags = provide_subword_tags
self.mode = kwargs.get('mode')
self.max_seq_length = max_seq_length
self.max_subword_length = max_subword_length
vocab_file = str(expand_path(vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
self.token_maksing_prob = token_maksing_prob
def __call__(self,
tokens: Union[List[List[str]], List[str]],
tags: List[List[str]] = None,
**kwargs):
if isinstance(tokens[0], str):
tokens = [re.findall(self._re_tokenizer, s) for s in tokens]
subword_tokens, subword_tok_ids, subword_masks, subword_tags = [], [], [], []
for i in range(len(tokens)):
toks = tokens[i]
ys = ['O'] * len(toks) if tags is None else tags[i]
mask = [int(y != 'X') for y in ys]
assert len(toks) == len(ys) == len(mask), \
f"toks({len(toks)}) should have the same length as " \
f" ys({len(ys)}) and mask({len(mask)}), tokens = {toks}."
sw_toks, sw_mask, sw_ys = self._ner_bert_tokenize(
toks,
mask,
ys,
self.tokenizer,
self.max_subword_length,
mode=self.mode,
token_maksing_prob=self.token_maksing_prob)
if self.max_seq_length is not None:
if len(sw_toks) > self.max_seq_length:
raise RuntimeError(
f"input sequence after bert tokenization"
f" shouldn't exceed {self.max_seq_length} tokens.")
subword_tokens.append(sw_toks)
subword_tok_ids.append(
self.tokenizer.convert_tokens_to_ids(sw_toks))
subword_masks.append(sw_mask)
subword_tags.append(sw_ys)
assert len(sw_mask) == len(sw_toks) == len(subword_tok_ids[-1]) == len(sw_ys), \
f"length of mask({len(sw_mask)}), tokens({len(sw_toks)})," \
f" token ids({len(subword_tok_ids[-1])}) and ys({len(ys)})" \
f" for tokens = `{toks}` should match"
subword_tok_ids = zero_pad(subword_tok_ids, dtype=int, padding=0)
subword_masks = zero_pad(subword_masks, dtype=int, padding=0)
if tags is not None:
if self.provide_subword_tags:
return tokens, subword_tokens, subword_tok_ids, subword_masks, subword_tags
else:
nonmasked_tags = [[t for t in ts if t != 'X'] for ts in tags]
for swts, swids, swms, ts in zip(subword_tokens,
subword_tok_ids,
subword_masks,
nonmasked_tags):
if (len(swids) != len(swms)) or (len(ts) != sum(swms)):
log.warning(
'Not matching lengths of the tokenization!')
log.warning(
f'Tokens len: {len(swts)}\n Tokens: {swts}')
log.warning(
f'Masks len: {len(swms)}, sum: {sum(swms)}')
log.warning(f'Masks: {swms}')
log.warning(f'Tags len: {len(ts)}\n Tags: {ts}')
return tokens, subword_tokens, subword_tok_ids, subword_masks, nonmasked_tags
return tokens, subword_tokens, subword_tok_ids, subword_masks
@staticmethod
def _ner_bert_tokenize(
tokens: List[str],
mask: List[int],
tags: List[str],
tokenizer: FullTokenizer,
max_subword_len: int = None,
mode: str = None,
token_maksing_prob: float = 0.0
) -> Tuple[List[str], List[int], List[str]]:
tokens_subword = ['[CLS]']
mask_subword = [0]
tags_subword = ['X']
for token, flag, tag in zip(tokens, mask, tags):
subwords = tokenizer.tokenize(token)
if not subwords or \
((max_subword_len is not None) and (len(subwords) > max_subword_len)):
tokens_subword.append('[UNK]')
mask_subword.append(flag)
tags_subword.append(tag)
else:
if mode == 'train' and token_maksing_prob > 0.0 and np.random.rand(
) < token_maksing_prob:
tokens_subword.extend(['[MASK]'] * len(subwords))
else:
tokens_subword.extend(subwords)
mask_subword.extend([flag] + [0] * (len(subwords) - 1))
tags_subword.extend([tag] + ['X'] * (len(subwords) - 1))
tokens_subword.append('[SEP]')
mask_subword.append(0)
tags_subword.append('X')
return tokens_subword, mask_subword, tags_subword
class BertNerPreprocessor(Component):
"""Takes tokens and splits them into bert subtokens, encode subtokens with their indices.
Creates mask of subtokens (one for first subtoken, zero for later subtokens).
If tags are provided, calculate tags for subtokens.
Args:
vocab_file: path to vocabulary
do_lower_case: set True if lowercasing is needed
max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
max_subword_length: replace token to <unk> if it's length is larger than this
(defaults to None, which is equal to +infinity)
Attributes:
max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
max_subword_length: rmax lenght of a bert subtoken
tokenizer: instance of Bert FullTokenizer
"""
def __init__(self,
vocab_file: str,
do_lower_case: bool = True,
max_seq_length: int = 512,
max_subword_length: int = None,
**kwargs):
self.max_seq_length = max_seq_length
self.max_subword_length = max_subword_length
vocab_file = str(expand_path(vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
def __call__(self,
tokens: List[List[str]],
tags: List[List[str]] = None,
**kwargs):
subword_tokens, subword_tok_ids, subword_masks, subword_tags = [], [], [], []
for i in range(len(tokens)):
toks = tokens[i]
ys = ['X'] * len(toks) if tags is None else tags[i]
assert len(toks) == len(ys), \
f"toks({len(toks)}) should have the same length as "\
f" ys({len(ys)}), tokens = {toks}."
sw_toks, sw_mask, sw_ys = self._ner_bert_tokenize(
toks, [1] * len(toks), ys, self.tokenizer,
self.max_subword_length)
if self.max_seq_length is not None:
sw_toks = sw_toks[:self.max_seq_length]
sw_mask = sw_mask[:self.max_seq_length]
sw_ys = sw_ys[:self.max_seq_length]
# add [sep] if we cut it
if sw_toks[-1] != '[SEP]':
sw_toks[-1] = '[SEP]'
sw_mask[-1] = 0
sw_ys[-1] = 'X'
subword_tokens.append(sw_toks)
subword_tok_ids.append(
self.tokenizer.convert_tokens_to_ids(sw_toks))
subword_masks.append(sw_mask)
subword_tags.append(sw_ys)
assert len(sw_mask) == len(sw_toks) == len(subword_tok_ids[-1]) == len(sw_ys),\
f"length of mask({len(sw_mask)}), tokens({len(sw_toks)}),"\
f" token ids({len(subword_tok_ids[-1])}) and ys({len(ys)})"\
f" for tokens = `{toks}` should match"
subword_tok_ids = zero_pad(subword_tok_ids, dtype=int, padding=0)
subword_masks = zero_pad(subword_masks, dtype=int, padding=0)
if tags is not None:
return subword_tokens, subword_tok_ids, subword_masks, subword_tags
return subword_tokens, subword_tok_ids, subword_masks
@staticmethod
def _ner_bert_tokenize(
tokens: List[str],
mask: List[int],
tags: List[str],
tokenizer: FullTokenizer,
max_subword_len: int = None) -> Tuple[List[str], List[str]]:
tokens_subword = ['[CLS]']
mask_subword = [0]
tags_subword = ['X']
for token, flag, tag in zip(tokens, mask, tags):
subwords = tokenizer.tokenize(token)
if not subwords or\
((max_subword_len is not None) and (len(subwords) > max_subword_len)):
tokens_subword.append('[UNK]')
mask_subword.append(0)
tags_subword.append('X')
else:
tokens_subword.extend(subwords)
mask_subword.extend([flag] + [0] * (len(subwords) - 1))
tags_subword.extend([tag] + ['X'] * (len(subwords) - 1))
tokens_subword.append('[SEP]')
mask_subword.append(0)
tags_subword.append('X')
return tokens_subword, mask_subword, tags_subword