def get_features(
queries,
max_seq_length,
tokenizer,
pad_label=128,
word_level_slots=None,
ignore_extra_tokens=False,
ignore_start_end=False,
):
"""
Convert queries (utterance, intent label and slot labels) to BERT input format
"""
all_subtokens = []
all_loss_mask = []
all_subtokens_mask = []
all_segment_ids = []
all_input_ids = []
all_input_mask = []
sent_lengths = []
all_slots = []
with_label = word_level_slots is not None
for i, query in enumerate(queries):
words = query.strip().split()
subtokens = [tokenizer.cls_token]
loss_mask = [1 - ignore_start_end]
subtokens_mask = [0]
if with_label:
slots = [pad_label]
for j, word in enumerate(words):
word_tokens = tokenizer.text_to_tokens(word)
# to handle emojis that could be neglected during tokenization
if len(word.strip()) > 0 and len(word_tokens) == 0:
word_tokens = [tokenizer.ids_to_tokens(tokenizer.unk_id)]
subtokens.extend(word_tokens)
# mask all sub-word tokens except the first token in a word
# use the label for the first sub-word token as the label for the entire word to eliminate need for disambiguation
loss_mask.append(1)
loss_mask.extend([int(not ignore_extra_tokens)] *
(len(word_tokens) - 1))
subtokens_mask.append(1)
subtokens_mask.extend([0] * (len(word_tokens) - 1))
if with_label:
slots.extend([word_level_slots[i][j]] * len(word_tokens))
subtokens.append(tokenizer.sep_token)
loss_mask.append(1 - ignore_start_end)
subtokens_mask.append(0)
sent_lengths.append(len(subtokens))
all_subtokens.append(subtokens)
all_loss_mask.append(loss_mask)
all_subtokens_mask.append(subtokens_mask)
all_input_mask.append([1] * len(subtokens))
if with_label:
slots.append(pad_label)
all_slots.append(slots)
max_seq_length_data = max(sent_lengths)
max_seq_length = min(
max_seq_length,
max_seq_length_data) if max_seq_length > 0 else max_seq_length_data
logging.info(f'Setting max length to: {max_seq_length}')
get_stats(sent_lengths)
# truncate and pad samples
(
all_slots,
all_subtokens,
all_input_mask,
all_loss_mask,
all_subtokens_mask,
all_input_ids,
all_segment_ids,
) = DialogueBERTDataset.truncate_and_pad(
max_seq_length,
ignore_start_end,
with_label,
pad_label,
tokenizer,
all_slots,
all_subtokens,
all_input_mask,
all_loss_mask,
all_subtokens_mask,
all_input_ids,
all_segment_ids,
)
# log examples for debugging
logging.debug("*** Some Examples of Processed Data ***")
for i in range(min(len(all_input_ids), 5)):
logging.debug("i: %s" % (i))
logging.debug("subtokens: %s" %
" ".join(list(map(str, all_subtokens[i]))))
logging.debug("loss_mask: %s" %
" ".join(list(map(str, all_loss_mask[i]))))
logging.debug("input_mask: %s" %
" ".join(list(map(str, all_input_mask[i]))))
logging.debug("subtokens_mask: %s" %
" ".join(list(map(str, all_subtokens_mask[i]))))
if with_label:
logging.debug("slots_label: %s" %
" ".join(list(map(str, all_slots[i]))))
return (all_input_ids, all_segment_ids, all_input_mask, all_loss_mask,
all_subtokens_mask, all_slots)
def get_features(
queries,
max_seq_length,
tokenizer,
pad_label=128,
raw_slots=None,
ignore_extra_tokens=False,
ignore_start_end=False,
):
all_subtokens = []
all_loss_mask = []
all_subtokens_mask = []
all_segment_ids = []
all_input_ids = []
all_input_mask = []
sent_lengths = []
all_slots = []
with_label = False
if raw_slots is not None:
with_label = True
for i, query in enumerate(queries):
words = query.strip().split()
subtokens = [tokenizer.cls_token]
loss_mask = [1 - ignore_start_end]
subtokens_mask = [0]
if with_label:
slots = [pad_label]
for j, word in enumerate(words):
word_tokens = tokenizer.text_to_tokens(word)
subtokens.extend(word_tokens)
loss_mask.append(1)
loss_mask.extend([int(not ignore_extra_tokens)] *
(len(word_tokens) - 1))
subtokens_mask.append(1)
subtokens_mask.extend([0] * (len(word_tokens) - 1))
if with_label:
slots.extend([raw_slots[i][j]] * len(word_tokens))
subtokens.append(tokenizer.sep_token)
loss_mask.append(1 - ignore_start_end)
subtokens_mask.append(0)
sent_lengths.append(len(subtokens))
all_subtokens.append(subtokens)
all_loss_mask.append(loss_mask)
all_subtokens_mask.append(subtokens_mask)
all_input_mask.append([1] * len(subtokens))
if with_label:
slots.append(pad_label)
all_slots.append(slots)
max_seq_length_data = max(sent_lengths)
max_seq_length = min(
max_seq_length,
max_seq_length_data) if max_seq_length > 0 else max_seq_length_data
logging.info(f'Setting max length to: {max_seq_length}')
get_stats(sent_lengths)
too_long_count = 0
for i, subtokens in enumerate(all_subtokens):
if len(subtokens) > max_seq_length:
subtokens = [tokenizer.cls_token] + subtokens[-max_seq_length + 1:]
all_input_mask[i] = [1] + all_input_mask[i][-max_seq_length + 1:]
all_loss_mask[i] = [1 - ignore_start_end
] + all_loss_mask[i][-max_seq_length + 1:]
all_subtokens_mask[i] = [
0
] + all_subtokens_mask[i][-max_seq_length + 1:]
if with_label:
all_slots[i] = [pad_label] + all_slots[i][-max_seq_length + 1:]
too_long_count += 1
all_input_ids.append([tokenizer.tokens_to_ids(t) for t in subtokens])
if len(subtokens) < max_seq_length:
extra = max_seq_length - len(subtokens)
all_input_ids[i] = all_input_ids[i] + [0] * extra
all_loss_mask[i] = all_loss_mask[i] + [0] * extra
all_subtokens_mask[i] = all_subtokens_mask[i] + [0] * extra
all_input_mask[i] = all_input_mask[i] + [0] * extra
if with_label:
all_slots[i] = all_slots[i] + [pad_label] * extra
all_segment_ids.append([0] * max_seq_length)
logging.info(f'{too_long_count} are longer than {max_seq_length}')
# May be useful for debugging
logging.debug("*** Some Examples of Processed Data ***")
for i in range(min(len(all_input_ids), 5)):
logging.debug("i: %s" % (i))
logging.debug("subtokens: %s" %
" ".join(list(map(str, all_subtokens[i]))))
logging.debug("loss_mask: %s" %
" ".join(list(map(str, all_loss_mask[i]))))
logging.debug("input_mask: %s" %
" ".join(list(map(str, all_input_mask[i]))))
logging.debug("subtokens_mask: %s" %
" ".join(list(map(str, all_subtokens_mask[i]))))
if with_label:
logging.debug("slots_label: %s" %
" ".join(list(map(str, all_slots[i]))))
return (all_input_ids, all_segment_ids, all_input_mask, all_loss_mask,
all_subtokens_mask, all_slots)