def get_features(all_sents,
tokenizer,
max_seq_length,
labels=None,
verbose=True):
"""Encode a list of sentences into a list of tuples of (input_ids, segment_ids, input_mask, label)."""
features = []
sent_lengths = []
too_long_count = 0
for sent_id, sent in enumerate(all_sents):
if sent_id % 1000 == 0:
logging.debug(f"Encoding sentence {sent_id}/{len(all_sents)}")
sent_subtokens = [tokenizer.cls_token]
for word in sent:
word_tokens = tokenizer.text_to_tokens(word)
sent_subtokens.extend(word_tokens)
if max_seq_length > 0 and len(sent_subtokens) + 1 > max_seq_length:
sent_subtokens = sent_subtokens[:max_seq_length]
too_long_count += 1
sent_subtokens.append(tokenizer.sep_token)
sent_lengths.append(len(sent_subtokens))
input_ids = [tokenizer.tokens_to_ids(t) for t in sent_subtokens]
# The mask has 1 for real tokens and 0 for padding tokens.
# Only real tokens are attended to.
input_mask = [1] * len(input_ids)
segment_ids = [0] * len(input_ids)
if verbose and sent_id < 2:
logging.info("*** Example ***")
logging.info(f"example {sent_id}: {sent}")
logging.info("subtokens: %s" % " ".join(sent_subtokens))
logging.info("input_ids: %s" % list2str(input_ids))
logging.info("segment_ids: %s" % list2str(segment_ids))
logging.info("input_mask: %s" % list2str(input_mask))
logging.info("label: %s" %
labels[sent_id] if labels else "**Not Provided**")
label = labels[sent_id] if labels else -1
features.append([
np.asarray(input_ids),
np.asarray(segment_ids),
np.asarray(input_mask), label
])
if max_seq_length > -1 and too_long_count > 0:
logging.warning(
f'Found {too_long_count} out of {len(all_sents)} sentences with more than {max_seq_length} subtokens. '
f'Truncated long sentences from the end.')
if verbose:
get_stats(sent_lengths)
return features
def get_features(
queries: List[str],
max_seq_length: int,
tokenizer: TokenizerSpec,
punct_label_ids: dict = None,
capit_label_ids: dict = None,
pad_label: str = 'O',
punct_labels_lines=None,
capit_labels_lines=None,
ignore_extra_tokens=False,
ignore_start_end: Optional[bool] = False,
):
"""
Processes the data and returns features.
Args:
queries: text sequences
max_seq_length: max sequence length minus 2 for [CLS] and [SEP]
tokenizer: such as AutoTokenizer
pad_label: pad value use for labels. By default, it's the neutral label.
punct_label_ids: dict to map punctuation labels to label ids.
Starts with pad_label->0 and then increases in alphabetical order.
Required for training and evaluation, not needed for inference.
capit_label_ids: dict to map labels to label ids. Starts
with pad_label->0 and then increases in alphabetical order.
Required for training and evaluation, not needed for inference.
punct_labels: list of labels for every word in a sequence (str)
capit_labels: list of labels for every word in a sequence (str)
ignore_extra_tokens: whether to ignore extra tokens in the loss_mask
ignore_start_end: whether to ignore bos and eos tokens in the loss_mask
Returns:
all_input_ids: input ids for all tokens
all_segment_ids: token type ids
all_input_mask: attention mask to use for BERT model
all_subtokens_mask: masks out all subwords besides the first one
all_loss_mask: loss mask to mask out tokens during training
punct_all_labels: all labels for punctuation task (ints)
capit_all_labels: all labels for capitalization task (ints)
punct_label_ids: label (str) to id (int) map for punctuation task
capit_label_ids: label (str) to id (int) map for capitalization task
"""
all_subtokens = []
all_loss_mask = []
all_subtokens_mask = []
all_segment_ids = []
all_input_ids = []
all_input_mask = []
sent_lengths = []
punct_all_labels = []
capit_all_labels = []
with_label = False
if punct_labels_lines and capit_labels_lines:
with_label = True
for i, query in enumerate(queries):
words = query.strip().split()
# add bos token
subtokens = [tokenizer.cls_token]
loss_mask = [1 - ignore_start_end]
subtokens_mask = [0]
if with_label:
pad_id = punct_label_ids[pad_label]
punct_labels = [pad_id]
punct_query_labels = [
punct_label_ids[lab] for lab in punct_labels_lines[i]
]
capit_labels = [pad_id]
capit_query_labels = [
capit_label_ids[lab] for lab in capit_labels_lines[i]
]
for j, word in enumerate(words):
word_tokens = tokenizer.text_to_tokens(word)
subtokens.extend(word_tokens)
loss_mask.append(1)
loss_mask.extend([int(not ignore_extra_tokens)] *
(len(word_tokens) - 1))
subtokens_mask.append(1)
subtokens_mask.extend([0] * (len(word_tokens) - 1))
if with_label:
punct_labels.extend([punct_query_labels[j]] * len(word_tokens))
capit_labels.extend([capit_query_labels[j]] * len(word_tokens))
# add eos token
subtokens.append(tokenizer.sep_token)
loss_mask.append(1 - ignore_start_end)
subtokens_mask.append(0)
sent_lengths.append(len(subtokens))
all_subtokens.append(subtokens)
all_loss_mask.append(loss_mask)
all_subtokens_mask.append(subtokens_mask)
all_input_mask.append([1] * len(subtokens))
if with_label:
punct_labels.append(pad_id)
punct_all_labels.append(punct_labels)
capit_labels.append(pad_id)
capit_all_labels.append(capit_labels)
max_seq_length = min(max_seq_length, max(sent_lengths))
logging.info(f'Max length: {max_seq_length}')
get_stats(sent_lengths)
too_long_count = 0
for i, subtokens in enumerate(all_subtokens):
if len(subtokens) > max_seq_length:
subtokens = [tokenizer.cls_token] + subtokens[-max_seq_length + 1:]
all_input_mask[i] = [1] + all_input_mask[i][-max_seq_length + 1:]
all_loss_mask[i] = [int(not ignore_start_end)
] + all_loss_mask[i][-max_seq_length + 1:]
all_subtokens_mask[i] = [
0
] + all_subtokens_mask[i][-max_seq_length + 1:]
if with_label:
punct_all_labels[i] = [
pad_id
] + punct_all_labels[i][-max_seq_length + 1:]
capit_all_labels[i] = [
pad_id
] + capit_all_labels[i][-max_seq_length + 1:]
too_long_count += 1
all_input_ids.append(tokenizer.tokens_to_ids(subtokens))
if len(subtokens) < max_seq_length:
extra = max_seq_length - len(subtokens)
all_input_ids[i] = all_input_ids[i] + [0] * extra
all_loss_mask[i] = all_loss_mask[i] + [0] * extra
all_subtokens_mask[i] = all_subtokens_mask[i] + [0] * extra
all_input_mask[i] = all_input_mask[i] + [0] * extra
if with_label:
punct_all_labels[i] = punct_all_labels[i] + [pad_id] * extra
capit_all_labels[i] = capit_all_labels[i] + [pad_id] * extra
all_segment_ids.append([0] * max_seq_length)
logging.info(f'{too_long_count} are longer than {max_seq_length}')
for i in range(min(len(all_input_ids), 5)):
logging.info("*** Example ***")
logging.info("i: %s" % (i))
logging.info("subtokens: %s" %
" ".join(list(map(str, all_subtokens[i]))))
logging.info("loss_mask: %s" %
" ".join(list(map(str, all_loss_mask[i]))))
logging.info("input_mask: %s" %
" ".join(list(map(str, all_input_mask[i]))))
logging.info("subtokens_mask: %s" %
" ".join(list(map(str, all_subtokens_mask[i]))))
if with_label:
logging.info("punct_labels: %s" %
" ".join(list(map(str, punct_all_labels[i]))))
logging.info("capit_labels: %s" %
" ".join(list(map(str, capit_all_labels[i]))))
return (
all_input_ids,
all_segment_ids,
all_input_mask,
all_subtokens_mask,
all_loss_mask,
punct_all_labels,
capit_all_labels,
punct_label_ids,
capit_label_ids,
)
def get_features_infer(
queries: List[str],
tokenizer: TokenizerSpec,
max_seq_length: int = 64,
step: Optional[int] = 8,
margin: Optional[int] = 16,
) -> Tuple[List[List[int]], List[List[int]], List[List[int]], List[List[int]],
List[int], List[int], List[bool], List[bool], ]:
"""
Processes the data and returns features.
Args:
queries: text sequences
tokenizer: such as AutoTokenizer
max_seq_length: max sequence length minus 2 for [CLS] and [SEP]
step: relative shift of consequent segments into which long queries are split. Long queries are split into
segments which can overlap. Parameter ``step`` controls such overlapping. Imagine that queries are
tokenized into characters, ``max_seq_length=5``, and ``step=2``. In such a case query "hello" is
tokenized into segments ``[['[CLS]', 'h', 'e', 'l', '[SEP]'], ['[CLS]', 'l', 'l', 'o', '[SEP]']]``.
margin: number of subtokens near edges of segments which are not used for punctuation and capitalization
prediction. The first segment does not have left margin and the last segment does not have right
margin. For example, if input sequence is tokenized into characters, ``max_seq_length=5``,
``step=1``, and ``margin=1``, then query "hello" will be tokenized into segments
``[['[CLS]', 'h', 'e', 'l', '[SEP]'], ['[CLS]', 'e', 'l', 'l', '[SEP]'],
['[CLS]', 'l', 'l', 'o', '[SEP]']]``. These segments are passed to the model. Before final predictions
computation, margins are removed. In the next list, subtokens which logits are not used for final
predictions computation are marked with asterisk: ``[['[CLS]'*, 'h', 'e', 'l'*, '[SEP]'*],
['[CLS]'*, 'e'*, 'l', 'l'*, '[SEP]'*], ['[CLS]'*, 'l'*, 'l', 'o', '[SEP]'*]]``.
Returns:
all_input_ids: list of input ids of all segments
all_segment_ids: token type ids of all segments
all_input_mask: attention mask to use for BERT model
all_subtokens_mask: masks out all subwords besides the first one
all_quantities_of_preceding_words: number of words in query preceding a segment. Used for joining
predictions from overlapping segments.
all_query_ids: index of a query to which segment belongs
all_is_first: is segment first segment in a query
all_is_last: is segment last segment in a query
"""
st = []
stm = []
sent_lengths = []
for i, query in enumerate(queries):
subtokens, subtokens_mask = _get_subtokens_and_subtokens_mask(
query, tokenizer)
sent_lengths.append(len(subtokens))
st.append(subtokens)
stm.append(subtokens_mask)
_check_max_seq_length_and_margin_and_step(max_seq_length, margin, step)
if max_seq_length > max(sent_lengths) + 2:
max_seq_length = max(sent_lengths) + 2
# If `max_seq_length` is greater than maximum length of input query, parameters ``margin`` and ``step`` are
# not used will not be used.
step = 1
# Maximum number of word subtokens in segment. The first and the last tokens in segment are CLS and EOS
length = max_seq_length - 2
else:
# Maximum number of word subtokens in segment. The first and the last tokens in segment are CLS and EOS
length = max_seq_length - 2
step = min(length - margin * 2, step)
logging.info(f'Max length: {max_seq_length}')
get_stats(sent_lengths)
all_input_ids, all_segment_ids, all_subtokens_mask, all_input_mask, all_input_mask = [], [], [], [], []
all_quantities_of_preceding_words, all_query_ids, all_is_first, all_is_last = [], [], [], []
for q_i, query_st in enumerate(st):
q_inp_ids, q_segment_ids, q_subtokens_mask, q_inp_mask, q_quantities_of_preceding_words = [], [], [], [], []
for i in range(0, max(len(query_st), length) - length + step, step):
subtokens = [tokenizer.cls_token
] + query_st[i:i + length] + [tokenizer.sep_token]
q_inp_ids.append(tokenizer.tokens_to_ids(subtokens))
q_segment_ids.append([0] * len(subtokens))
q_subtokens_mask.append([0] + stm[q_i][i:i + length] + [0])
q_inp_mask.append([1] * len(subtokens))
q_quantities_of_preceding_words.append(
np.count_nonzero(stm[q_i][:i]))
all_input_ids.append(q_inp_ids)
all_segment_ids.append(q_segment_ids)
all_subtokens_mask.append(q_subtokens_mask)
all_input_mask.append(q_inp_mask)
all_quantities_of_preceding_words.append(
q_quantities_of_preceding_words)
all_query_ids.append([q_i] * len(q_inp_ids))
all_is_first.append([True] + [False] * (len(q_inp_ids) - 1))
all_is_last.append([False] * (len(q_inp_ids) - 1) + [True])
return (
list(itertools.chain(*all_input_ids)),
list(itertools.chain(*all_segment_ids)),
list(itertools.chain(*all_input_mask)),
list(itertools.chain(*all_subtokens_mask)),
list(itertools.chain(*all_quantities_of_preceding_words)),
list(itertools.chain(*all_query_ids)),
list(itertools.chain(*all_is_first)),
list(itertools.chain(*all_is_last)),
)
def get_features(
queries: List[str],
tokenizer: TokenizerSpec,
max_seq_length: int = -1,
label_ids: dict = None,
pad_label: str = 'O',
raw_labels: List[str] = None,
ignore_extra_tokens: bool = False,
ignore_start_end: bool = False,
):
"""
Processes the data and returns features.
Args:
queries: text sequences
tokenizer: such as AutoTokenizer
max_seq_length: max sequence length minus 2 for [CLS] and [SEP], when -1 - use the max len from the data
pad_label: pad value use for labels. By default, it's the neutral label.
raw_labels: list of labels for every word in a sequence
label_ids: dict to map labels to label ids.
Starts with pad_label->0 and then increases in alphabetical order.
Required for training and evaluation, not needed for inference.
ignore_extra_tokens: whether to ignore extra tokens in the loss_mask
ignore_start_end: whether to ignore bos and eos tokens in the loss_mask
"""
all_subtokens = []
all_loss_mask = []
all_subtokens_mask = []
all_segment_ids = []
all_input_ids = []
all_input_mask = []
sent_lengths = []
all_labels = []
with_label = False
if raw_labels is not None:
with_label = True
for i, query in enumerate(queries):
words = query.strip().split()
# add bos token
subtokens = [tokenizer.cls_token]
loss_mask = [1 - ignore_start_end]
subtokens_mask = [0]
if with_label:
pad_id = label_ids[pad_label]
labels = [pad_id]
query_labels = [label_ids[lab] for lab in raw_labels[i]]
for j, word in enumerate(words):
word_tokens = tokenizer.text_to_tokens(word)
# to handle emojis that could be neglected during tokenization
if len(word.strip()) > 0 and len(word_tokens) == 0:
word_tokens = [tokenizer.ids_to_tokens(tokenizer.unk_id)]
subtokens.extend(word_tokens)
loss_mask.append(1)
loss_mask.extend([int(not ignore_extra_tokens)] * (len(word_tokens) - 1))
subtokens_mask.append(1)
subtokens_mask.extend([0] * (len(word_tokens) - 1))
if with_label:
labels.extend([query_labels[j]] * len(word_tokens))
# add eos token
subtokens.append(tokenizer.sep_token)
loss_mask.append(1 - ignore_start_end)
subtokens_mask.append(0)
sent_lengths.append(len(subtokens))
all_subtokens.append(subtokens)
all_loss_mask.append(loss_mask)
all_subtokens_mask.append(subtokens_mask)
all_input_mask.append([1] * len(subtokens))
if with_label:
labels.append(pad_id)
all_labels.append(labels)
max_seq_length_data = max(sent_lengths)
max_seq_length = min(max_seq_length, max_seq_length_data) if max_seq_length > 0 else max_seq_length_data
logging.info(f'Setting Max Seq length to: {max_seq_length}')
get_stats(sent_lengths)
too_long_count = 0
for i, subtokens in enumerate(all_subtokens):
if len(subtokens) > max_seq_length:
subtokens = [tokenizer.cls_token] + subtokens[-max_seq_length + 1 :]
all_input_mask[i] = [1] + all_input_mask[i][-max_seq_length + 1 :]
all_loss_mask[i] = [int(not ignore_start_end)] + all_loss_mask[i][-max_seq_length + 1 :]
all_subtokens_mask[i] = [0] + all_subtokens_mask[i][-max_seq_length + 1 :]
if with_label:
all_labels[i] = [pad_id] + all_labels[i][-max_seq_length + 1 :]
too_long_count += 1
all_input_ids.append(tokenizer.tokens_to_ids(subtokens))
if len(subtokens) < max_seq_length:
extra = max_seq_length - len(subtokens)
all_input_ids[i] = all_input_ids[i] + [0] * extra
all_loss_mask[i] = all_loss_mask[i] + [0] * extra
all_subtokens_mask[i] = all_subtokens_mask[i] + [0] * extra
all_input_mask[i] = all_input_mask[i] + [0] * extra
if with_label:
all_labels[i] = all_labels[i] + [pad_id] * extra
all_segment_ids.append([0] * max_seq_length)
logging.warning(f'{too_long_count} are longer than {max_seq_length}')
for i in range(min(len(all_input_ids), 1)):
logging.info("*** Example ***")
logging.info("i: %s", i)
logging.info("subtokens: %s", " ".join(list(map(str, all_subtokens[i]))))
logging.info("loss_mask: %s", " ".join(list(map(str, all_loss_mask[i]))))
logging.info("input_mask: %s", " ".join(list(map(str, all_input_mask[i]))))
logging.info("subtokens_mask: %s", " ".join(list(map(str, all_subtokens_mask[i]))))
if with_label:
logging.info("labels: %s", " ".join(list(map(str, all_labels[i]))))
return (all_input_ids, all_segment_ids, all_input_mask, all_subtokens_mask, all_loss_mask, all_labels)
