def __init__(self,
squad_model_config: str,
vocab_file: str,
do_lower_case: bool,
max_seq_length: int = 512,
batch_size: int = 10,
lang='en',
**kwargs) -> None:
config = json.load(open(squad_model_config))
config['chainer']['pipe'][0]['max_seq_length'] = max_seq_length
self.model = build_model(config)
self.max_seq_length = max_seq_length
vocab_file = str(expand_path(vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
self.batch_size = batch_size
if lang == 'en':
from nltk import sent_tokenize
self.sent_tokenizer = sent_tokenize
elif lang == 'ru':
from ru_sent_tokenize import ru_sent_tokenize
self.sent_tokenizer = ru_sent_tokenize
else:
raise RuntimeError('en and ru languages are supported only')
def __init__(self,
vocab_file: str,
do_lower_case: bool = True,
max_seq_length: int = 512,
max_subword_length: int = None,
**kwargs):
self.max_seq_length = max_seq_length
self.max_subword_length = max_subword_length
vocab_file = str(expand_path(vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
def _ner_bert_tokenize(
tokens: List[str],
mask: List[int],
tags: List[str],
tokenizer: FullTokenizer,
max_subword_len: int = None,
mode: str = None,
token_maksing_prob: float = 0.0
) -> Tuple[List[str], List[int], List[str]]:
tokens_subword = ['[CLS]']
mask_subword = [0]
tags_subword = ['X']
for token, flag, tag in zip(tokens, mask, tags):
subwords = tokenizer.tokenize(token)
if not subwords or \
((max_subword_len is not None) and (len(subwords) > max_subword_len)):
tokens_subword.append('[UNK]')
mask_subword.append(flag)
tags_subword.append(tag)
else:
if mode == 'train' and token_maksing_prob > 0.0 and np.random.rand(
) < token_maksing_prob:
tokens_subword.extend(['[MASK]'] * len(subwords))
else:
tokens_subword.extend(subwords)
mask_subword.extend([flag] + [0] * (len(subwords) - 1))
tags_subword.extend([tag] + ['X'] * (len(subwords) - 1))
tokens_subword.append('[SEP]')
mask_subword.append(0)
tags_subword.append('X')
return tokens_subword, mask_subword, tags_subword
def _ner_bert_tokenize(
tokens: List[str],
mask: List[int],
tags: List[str],
tokenizer: FullTokenizer,
max_subword_len: int = None) -> Tuple[List[str], List[str]]:
tokens_subword = ['[CLS]']
mask_subword = [0]
tags_subword = ['X']
for token, flag, tag in zip(tokens, mask, tags):
subwords = tokenizer.tokenize(token)
if not subwords or\
((max_subword_len is not None) and (len(subwords) > max_subword_len)):
tokens_subword.append('[UNK]')
mask_subword.append(0)
tags_subword.append('X')
else:
tokens_subword.extend(subwords)
mask_subword.extend([flag] + [0] * (len(subwords) - 1))
tags_subword.extend([tag] + ['X'] * (len(subwords) - 1))
tokens_subword.append('[SEP]')
mask_subword.append(0)
tags_subword.append('X')
return tokens_subword, mask_subword, tags_subword
def __init__(self,
data: Dict[str, List[Tuple[Any, Any]]],
bert_tokenizer_vocab_file: str,
do_lower_case: bool = False,
left_context_rate: float = 0.5,
max_seq_length: int = None,
one_sample_per_doc: bool = False,
seed: int = None,
shuffle: bool = True,
*args,
**kwargs) -> None:
self.max_seq_length = max_seq_length or float('inf')
self.one_sample_per_doc = one_sample_per_doc
self.left_context_rate = left_context_rate
self.shuffle = shuffle
vocab_file = str(expand_path(bert_tokenizer_vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
self.random = Random(seed)
self.train = data.get('train', [])
self.valid = data.get('valid', [])
self.test = data.get('test', [])
self.split(*args, **kwargs)
self.data = {
'train': self.train,
'valid': self.valid,
'test': self.test,
'all': self.train + self.test + self.valid
}
def __init__(self,
vocab_file: str,
do_lower_case: bool = True,
max_seq_length: int = 512,
max_subword_length: int = None,
token_maksing_prob: float = 0.0,
provide_subword_tags: bool = False,
**kwargs):
self._re_tokenizer = re.compile(r"[\w']+|[^\w ]")
self.provide_subword_tags = provide_subword_tags
self.mode = kwargs.get('mode')
self.max_seq_length = max_seq_length
self.max_subword_length = max_subword_length
vocab_file = str(expand_path(vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
self.token_maksing_prob = token_maksing_prob
def get_context_indices(
samples: List[List[str]],
sample_id: int,
subtokenizer: FullTokenizer,
max_subtokens_length: int,
left_context_rate: float = 0.5,
random: Random = Random(31)) -> List[int]:
rich_sample_indices = [sample_id]
toks = samples[sample_id]
l_ctx = samples[:sample_id]
r_ctx = samples[sample_id + 1:]
subtoks_len = len(
[st for t in toks for st in subtokenizer.tokenize(t)])
l_i, r_i = 0, 0
while (l_i < len(l_ctx)) or (r_i < len(r_ctx)):
l_rate = left_context_rate if r_i < len(r_ctx) else 1.0
if (l_i < len(l_ctx)) and (random.random() < l_rate):
# add one sentence from left_context
subtoks = [
st for t in l_ctx[-l_i - 1]
for st in subtokenizer.tokenize(t)
]
if subtoks_len + len(subtoks) > max_subtokens_length:
break
subtoks_len += len(subtoks)
rich_sample_indices = [sample_id - l_i - 1
] + rich_sample_indices
l_i += 1
else:
# add one sentence from right_context
subtoks = [
st for t in r_ctx[r_i] for st in subtokenizer.tokenize(t)
]
if subtoks_len + len(subtoks) > max_subtokens_length:
break
subtoks_len += len(subtoks)
rich_sample_indices.append(sample_id + r_i + 1)
r_i += 1
return rich_sample_indices
def __init__(self,
data: Dict[str, List[Tuple[Any, Any]]],
bert_tokenizer_vocab_file: str,
do_lower_case: bool = False,
left_context_rate: float = 0.5,
max_seq_length: int = None,
one_sample_per_doc: bool = False,
seed: int = None,
shuffle: bool = True,
*args,
**kwargs) -> None:
self.max_seq_length = max_seq_length or float('inf')
self.one_sample_per_doc = one_sample_per_doc
self.left_context_rate = left_context_rate
vocab_file = str(expand_path(bert_tokenizer_vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
super().__init__(data, seed, shuffle, *args, **kwargs)
def __init__(self,
vocab_file,
do_lower_case=True,
max_seq_length: int = 512,
resps=None,
resp_vecs=None,
conts=None,
cont_vecs=None,
*args,
**kwargs):
self.max_seq_length = max_seq_length
vocab_file = str(expand_path(vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
self.resp_features = None
self.cont_features = None
if resps is not None and resp_vecs is None:
resp_batch = [[el] for el in resps]
self.resp_features = self(resp_batch)
if conts is not None and cont_vecs is None:
cont_batch = [[el] for el in conts]
self.cont_features = self(cont_batch)
def _ner_bert_tokenize(
tokens: List[str],
tags: List[str],
tokenizer: FullTokenizer,
max_subword_len: int = None,
mode: str = None,
subword_mask_mode: str = "first",
token_masking_prob: float = None
) -> Tuple[List[str], List[int], List[str]]:
do_masking = (mode == 'train') and (token_masking_prob is not None)
do_cutting = (max_subword_len is not None)
tokens_subword = ['[CLS]']
startofword_markers = [0]
tags_subword = ['X']
for token, tag in zip(tokens, tags):
token_marker = int(tag != 'X')
subwords = tokenizer.tokenize(token)
if not subwords or (do_cutting and
(len(subwords) > max_subword_len)):
tokens_subword.append('[UNK]')
startofword_markers.append(token_marker)
tags_subword.append(tag)
else:
if do_masking and (random.random() < token_masking_prob):
tokens_subword.extend(['[MASK]'] * len(subwords))
else:
tokens_subword.extend(subwords)
if subword_mask_mode == "last":
startofword_markers.extend([0] * (len(subwords) - 1) +
[token_marker])
else:
startofword_markers.extend([token_marker] + [0] *
(len(subwords) - 1))
tags_subword.extend([tag] + ['X'] * (len(subwords) - 1))
tokens_subword.append('[SEP]')
startofword_markers.append(0)
tags_subword.append('X')
return tokens_subword, startofword_markers, tags_subword
class BertSQuADInferModel(Component):
"""This model wraps BertSQuADModel to make predictions on longer than 512 tokens sequences.
It splits context on chunks with `max_seq_length - 3 - len(question)` length, preserving sentences boundaries.
It reassembles batches with chunks instead of full contexts to optimize performance, e.g.,:
batch_size = 5
number_of_contexts == 2
number of first context chunks == 8
number of second context chunks == 2
we will create two batches with 5 chunks
For each context the best answer is selected via logits or scores from BertSQuADModel.
Args:
squad_model_config: path to DeepPavlov BertSQuADModel config file
vocab_file: path to Bert vocab file
do_lower_case: set True if lowercasing is needed
max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
batch_size: size of batch to use during inference
lang: either `en` or `ru`, it is used to select sentence tokenizer
"""
def __init__(self, squad_model_config: str,
vocab_file: str,
do_lower_case: bool,
max_seq_length: int = 512,
batch_size: int = 10,
lang='en', **kwargs) -> None:
config = json.load(open(squad_model_config))
config['chainer']['pipe'][0]['max_seq_length'] = max_seq_length
self.model = build_model(config)
self.max_seq_length = max_seq_length
vocab_file = str(expand_path(vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file, do_lower_case=do_lower_case)
self.batch_size = batch_size
if lang == 'en':
from nltk import sent_tokenize
self.sent_tokenizer = sent_tokenize
elif lang == 'ru':
from ru_sent_tokenize import ru_sent_tokenize
self.sent_tokenizer = ru_sent_tokenize
else:
raise RuntimeError('en and ru languages are supported only')
def __call__(self, contexts: List[str], questions: List[str], **kwargs) -> Tuple[List[str], List[int], List[float]]:
"""get predictions for given contexts and questions
Args:
contexts: batch of contexts
questions: batch of questions
Returns:
predictions: answer, answer start position, logits or scores
"""
batch_indices = []
contexts_to_predict = []
questions_to_predict = []
predictions = {}
for i, (context, question) in enumerate(zip(contexts, questions)):
context_subtokens = self.tokenizer.tokenize(context)
question_subtokens = self.tokenizer.tokenize(question)
max_chunk_len = self.max_seq_length - len(question_subtokens) - 3
if 0 < max_chunk_len < len(context_subtokens):
number_of_chunks = math.ceil(len(context_subtokens) / max_chunk_len)
sentences = self.sent_tokenizer(context)
for chunk in np.array_split(sentences, number_of_chunks):
contexts_to_predict += [' '.join(chunk)]
questions_to_predict += [question]
batch_indices += [i]
else:
contexts_to_predict += [context]
questions_to_predict += [question]
batch_indices += [i]
for j in range(0, len(contexts_to_predict), self.batch_size):
c_batch = contexts_to_predict[j: j + self.batch_size]
q_batch = questions_to_predict[j: j + self.batch_size]
ind_batch = batch_indices[j: j + self.batch_size]
a_batch, a_st_batch, logits_batch = self.model(c_batch, q_batch)
for a, a_st, logits, ind in zip(a_batch, a_st_batch, logits_batch, ind_batch):
if ind in predictions:
predictions[ind] += [(a, a_st, logits)]
else:
predictions[ind] = [(a, a_st, logits)]
answers, answer_starts, logits = [], [], []
for ind in sorted(predictions.keys()):
prediction = predictions[ind]
best_answer_ind = np.argmax([p[2] for p in prediction])
answers += [prediction[best_answer_ind][0]]
answer_starts += [prediction[best_answer_ind][1]]
logits += [prediction[best_answer_ind][2]]
return answers, answer_starts, logits
class BertNerPreprocessor(Component):
"""Takes tokens and splits them into bert subtokens, encode subtokens with their indices.
Creates mask of subtokens (one for first subtoken, zero for later subtokens).
If tags are provided, calculate tags for subtokens.
Args:
vocab_file: path to vocabulary
do_lower_case: set True if lowercasing is needed
max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
max_subword_length: replace token to <unk> if it's length is larger than this
(defaults to None, which is equal to +infinity)
token_mask_prob: probability of masking token while training
provide_subword_tags: output tags for subwords or for words
Attributes:
max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
max_subword_length: rmax lenght of a bert subtoken
tokenizer: instance of Bert FullTokenizer
"""
def __init__(self,
vocab_file: str,
do_lower_case: bool = True,
max_seq_length: int = 512,
max_subword_length: int = None,
token_maksing_prob: float = 0.0,
provide_subword_tags: bool = False,
**kwargs):
self._re_tokenizer = re.compile(r"[\w']+|[^\w ]")
self.provide_subword_tags = provide_subword_tags
self.mode = kwargs.get('mode')
self.max_seq_length = max_seq_length
self.max_subword_length = max_subword_length
vocab_file = str(expand_path(vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
self.token_maksing_prob = token_maksing_prob
def __call__(self,
tokens: Union[List[List[str]], List[str]],
tags: List[List[str]] = None,
**kwargs):
if isinstance(tokens[0], str):
tokens = [re.findall(self._re_tokenizer, s) for s in tokens]
subword_tokens, subword_tok_ids, subword_masks, subword_tags = [], [], [], []
for i in range(len(tokens)):
toks = tokens[i]
ys = ['O'] * len(toks) if tags is None else tags[i]
mask = [int(y != 'X') for y in ys]
assert len(toks) == len(ys) == len(mask), \
f"toks({len(toks)}) should have the same length as " \
f" ys({len(ys)}) and mask({len(mask)}), tokens = {toks}."
sw_toks, sw_mask, sw_ys = self._ner_bert_tokenize(
toks,
mask,
ys,
self.tokenizer,
self.max_subword_length,
mode=self.mode,
token_maksing_prob=self.token_maksing_prob)
if self.max_seq_length is not None:
if len(sw_toks) > self.max_seq_length:
raise RuntimeError(
f"input sequence after bert tokenization"
f" shouldn't exceed {self.max_seq_length} tokens.")
subword_tokens.append(sw_toks)
subword_tok_ids.append(
self.tokenizer.convert_tokens_to_ids(sw_toks))
subword_masks.append(sw_mask)
subword_tags.append(sw_ys)
assert len(sw_mask) == len(sw_toks) == len(subword_tok_ids[-1]) == len(sw_ys), \
f"length of mask({len(sw_mask)}), tokens({len(sw_toks)})," \
f" token ids({len(subword_tok_ids[-1])}) and ys({len(ys)})" \
f" for tokens = `{toks}` should match"
subword_tok_ids = zero_pad(subword_tok_ids, dtype=int, padding=0)
subword_masks = zero_pad(subword_masks, dtype=int, padding=0)
if tags is not None:
if self.provide_subword_tags:
return tokens, subword_tokens, subword_tok_ids, subword_masks, subword_tags
else:
nonmasked_tags = [[t for t in ts if t != 'X'] for ts in tags]
for swts, swids, swms, ts in zip(subword_tokens,
subword_tok_ids,
subword_masks,
nonmasked_tags):
if (len(swids) != len(swms)) or (len(ts) != sum(swms)):
log.warning(
'Not matching lengths of the tokenization!')
log.warning(
f'Tokens len: {len(swts)}\n Tokens: {swts}')
log.warning(
f'Masks len: {len(swms)}, sum: {sum(swms)}')
log.warning(f'Masks: {swms}')
log.warning(f'Tags len: {len(ts)}\n Tags: {ts}')
return tokens, subword_tokens, subword_tok_ids, subword_masks, nonmasked_tags
return tokens, subword_tokens, subword_tok_ids, subword_masks
@staticmethod
def _ner_bert_tokenize(
tokens: List[str],
mask: List[int],
tags: List[str],
tokenizer: FullTokenizer,
max_subword_len: int = None,
mode: str = None,
token_maksing_prob: float = 0.0
) -> Tuple[List[str], List[int], List[str]]:
tokens_subword = ['[CLS]']
mask_subword = [0]
tags_subword = ['X']
for token, flag, tag in zip(tokens, mask, tags):
subwords = tokenizer.tokenize(token)
if not subwords or \
((max_subword_len is not None) and (len(subwords) > max_subword_len)):
tokens_subword.append('[UNK]')
mask_subword.append(flag)
tags_subword.append(tag)
else:
if mode == 'train' and token_maksing_prob > 0.0 and np.random.rand(
) < token_maksing_prob:
tokens_subword.extend(['[MASK]'] * len(subwords))
else:
tokens_subword.extend(subwords)
mask_subword.extend([flag] + [0] * (len(subwords) - 1))
tags_subword.extend([tag] + ['X'] * (len(subwords) - 1))
tokens_subword.append('[SEP]')
mask_subword.append(0)
tags_subword.append('X')
return tokens_subword, mask_subword, tags_subword
class BertNerPreprocessor(Component):
"""Takes tokens and splits them into bert subtokens, encode subtokens with their indices.
Creates mask of subtokens (one for first subtoken, zero for later subtokens).
If tags are provided, calculate tags for subtokens.
Args:
vocab_file: path to vocabulary
do_lower_case: set True if lowercasing is needed
max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
max_subword_length: replace token to <unk> if it's length is larger than this
(defaults to None, which is equal to +infinity)
Attributes:
max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
max_subword_length: rmax lenght of a bert subtoken
tokenizer: instance of Bert FullTokenizer
"""
def __init__(self,
vocab_file: str,
do_lower_case: bool = True,
max_seq_length: int = 512,
max_subword_length: int = None,
**kwargs):
self.max_seq_length = max_seq_length
self.max_subword_length = max_subword_length
vocab_file = str(expand_path(vocab_file))
self.tokenizer = FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
def __call__(self,
tokens: List[List[str]],
tags: List[List[str]] = None,
**kwargs):
subword_tokens, subword_tok_ids, subword_masks, subword_tags = [], [], [], []
for i in range(len(tokens)):
toks = tokens[i]
ys = ['X'] * len(toks) if tags is None else tags[i]
assert len(toks) == len(ys), \
f"toks({len(toks)}) should have the same length as "\
f" ys({len(ys)}), tokens = {toks}."
sw_toks, sw_mask, sw_ys = self._ner_bert_tokenize(
toks, [1] * len(toks), ys, self.tokenizer,
self.max_subword_length)
if self.max_seq_length is not None:
sw_toks = sw_toks[:self.max_seq_length]
sw_mask = sw_mask[:self.max_seq_length]
sw_ys = sw_ys[:self.max_seq_length]
# add [sep] if we cut it
if sw_toks[-1] != '[SEP]':
sw_toks[-1] = '[SEP]'
sw_mask[-1] = 0
sw_ys[-1] = 'X'
subword_tokens.append(sw_toks)
subword_tok_ids.append(
self.tokenizer.convert_tokens_to_ids(sw_toks))
subword_masks.append(sw_mask)
subword_tags.append(sw_ys)
assert len(sw_mask) == len(sw_toks) == len(subword_tok_ids[-1]) == len(sw_ys),\
f"length of mask({len(sw_mask)}), tokens({len(sw_toks)}),"\
f" token ids({len(subword_tok_ids[-1])}) and ys({len(ys)})"\
f" for tokens = `{toks}` should match"
subword_tok_ids = zero_pad(subword_tok_ids, dtype=int, padding=0)
subword_masks = zero_pad(subword_masks, dtype=int, padding=0)
if tags is not None:
return subword_tokens, subword_tok_ids, subword_masks, subword_tags
return subword_tokens, subword_tok_ids, subword_masks
@staticmethod
def _ner_bert_tokenize(
tokens: List[str],
mask: List[int],
tags: List[str],
tokenizer: FullTokenizer,
max_subword_len: int = None) -> Tuple[List[str], List[str]]:
tokens_subword = ['[CLS]']
mask_subword = [0]
tags_subword = ['X']
for token, flag, tag in zip(tokens, mask, tags):
subwords = tokenizer.tokenize(token)
if not subwords or\
((max_subword_len is not None) and (len(subwords) > max_subword_len)):
tokens_subword.append('[UNK]')
mask_subword.append(0)
tags_subword.append('X')
else:
tokens_subword.extend(subwords)
mask_subword.extend([flag] + [0] * (len(subwords) - 1))
tags_subword.extend([tag] + ['X'] * (len(subwords) - 1))
tokens_subword.append('[SEP]')
mask_subword.append(0)
tags_subword.append('X')
return tokens_subword, mask_subword, tags_subword
