def samples_to_features_bert_lm(sample,
max_seq_len,
tokenizer,
next_sent_pred=True):
"""
Convert a raw sample (pair of sentences as tokenized strings) into a proper training sample with
IDs, LM labels, padding_mask, CLS and SEP tokens etc.
:param sample: Sample, containing sentence input as strings and is_next label
:type sample: Sample
:param max_seq_len: Maximum length of sequence.
:type max_seq_len: int
:param tokenizer: Tokenizer
:return: InputFeatures, containing all inputs and labels of one sample as IDs (as used for model training)
"""
tokens_a = sample.tokenized["text_a"]["tokens"]
tokens_b = sample.tokenized["text_b"]["tokens"]
# Modifies `tokens_a` and `tokens_b` in place so that the total
# length is less than the specified length.
# Account for [CLS], [SEP], [SEP] or [CLS], [SEP]
if not next_sent_pred:
n_special_tokens = 2
else:
n_special_tokens = 3
truncate_seq_pair(tokens_a, tokens_b, max_seq_len - n_special_tokens)
tokens_a, t1_label = mask_random_words(
tokens_a,
tokenizer.vocab,
token_groups=sample.tokenized["text_a"]["start_of_word"])
# convert lm labels to ids
t1_label_ids = [
-1 if tok == '' else tokenizer.vocab[tok] for tok in t1_label
]
if next_sent_pred:
tokens_b, t2_label = mask_random_words(
tokens_b,
tokenizer.vocab,
token_groups=sample.tokenized["text_b"]["start_of_word"])
t2_label_ids = [
-1 if tok == '' else tokenizer.vocab[tok] for tok in t2_label
]
# concatenate lm labels and account for CLS, SEP, SEP
lm_label_ids = [-1] + t1_label_ids + [-1] + t2_label_ids + [-1]
else:
lm_label_ids = [-1] + t1_label_ids + [-1]
# The convention in BERT is:
# (a) For sequence pairs:
# tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
# type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1
# (b) For single sequences:
# tokens: [CLS] the dog is hairy . [SEP]
# type_ids: 0 0 0 0 0 0 0
#
# Where "type_ids" are used to indicate whether this is the first
# sequence or the second sequence. The embedding vectors for `type=0` and
# `type=1` were learned during pre-training and are added to the wordpiece
# embedding vector (and position vector). This is not *strictly* necessary
# since the [SEP] token unambigiously separates the sequences, but it makes
# it easier for the model to learn the concept of sequences.
#
# For classification tasks, the first vector (corresponding to [CLS]) is
# used as as the "sentence vector". Note that this only makes sense because
# the entire model is fine-tuned.
tokens = []
segment_ids = []
tokens.append("[CLS]")
segment_ids.append(0)
for token in tokens_a:
tokens.append(token)
segment_ids.append(0)
tokens.append("[SEP]")
segment_ids.append(0)
if next_sent_pred:
assert len(tokens_b) > 0
for token in tokens_b:
tokens.append(token)
segment_ids.append(1)
tokens.append("[SEP]")
segment_ids.append(1)
input_ids = tokenizer.convert_tokens_to_ids(tokens)
# The mask has 1 for real tokens and 0 for padding tokens. Only real
# tokens are attended to.
padding_mask = [1] * len(input_ids)
# Zero-pad up to the sequence length.
while len(input_ids) < max_seq_len:
input_ids.append(0)
padding_mask.append(0)
segment_ids.append(0)
lm_label_ids.append(-1)
# Convert is_next_label: Note that in Bert, is_next_labelid = 0 is used for next_sentence=true!
if next_sent_pred:
if sample.clear_text["nextsentence_label"]:
is_next_label_id = [0]
else:
is_next_label_id = [1]
assert len(input_ids) == max_seq_len
assert len(padding_mask) == max_seq_len
assert len(segment_ids) == max_seq_len
assert len(lm_label_ids) == max_seq_len
feature_dict = {
"input_ids": input_ids,
"padding_mask": padding_mask,
"segment_ids": segment_ids,
"lm_label_ids": lm_label_ids,
}
if next_sent_pred:
feature_dict["nextsentence_label_ids"] = is_next_label_id
return [feature_dict]
def premasked_samples_with_answers_to_features_bert_char_mlm(
sample, max_seq_len, tokenizer):
"""
This method is a copy of samples_to_features_bert_lm from farm/data_handler/input_features.py. It has been modified to not mask the samples, but simply convert the existing masking. It expects the sample texts to consist of the text then the answers separated by a tab. This is only used when the text has been masked by an external masking algorithm. -BN
Original documentation:
Convert a raw sample (pair of sentences as tokenized strings) into a proper training sample with
IDs, LM labels, padding_mask, CLS and SEP tokens etc.
:param sample: Sample, containing sentence input as strings and is_next label
:param max_seq_len: int, maximum length of sequence.
:param tokenizer: Tokenizer
:return: InputFeatures, containing all inputs and labels of one sample as IDs (as used for model training)
"""
tokens_a = sample.tokenized["text_a"]["tokens"]
tokens_b = sample.tokenized["text_b"]["tokens"]
seq_and_ans = "".join(tokens_a).split("\t")
tokens_a = seq_and_ans[0]
ans = seq_and_ans[1]
# usually t1_label and t2_label would contain the original unmasked tokens, here to have to construct t1 from the answers, t2 is just a copy of the placeholder
t1_label = tokens_a
t2_label = tokens_b.copy()
# construct t1_label
for c in ans:
t1_label = t1_label.replace("#", c, 1)
# here we're effectively retokenizing...
tokens_a = list(tokens_a)
t1_label = list(t1_label)
# Modifies `tokens_a` and `tokens_b` in place so that the total
# length is less than the specified length.
# Account for [CLS], [SEP], [SEP] with "- 3"
truncate_seq_pair(tokens_a, tokens_b, max_seq_len - 3)
# convert masking
conversions = 0
for i, t in enumerate(tokens_a):
if t == "#":
tokens_a[i] = "[MASK]"
conversions += 1
assert conversions == len(ans)
# remove unknown tokens
tokens_a = remove_unknown_chars(tokens_a, tokenizer)
t1_label = remove_unknown_chars(t1_label, tokenizer)
# convert lm labels to ids
t1_label_ids = [
-1 if tok == "" else tokenizer.vocab[tok] for tok in t1_label
]
t2_label_ids = [
-1 if tok == "" else tokenizer.vocab[tok] for tok in t2_label
]
# concatenate lm labels and account for CLS, SEP, SEP
lm_label_ids = [-1] + t1_label_ids + [-1] + t2_label_ids + [-1]
# The convention in BERT is:
# (a) For sequence pairs:
# tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
# type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1
# (b) For single sequences:
# tokens: [CLS] the dog is hairy . [SEP]
# type_ids: 0 0 0 0 0 0 0
#
# Where "type_ids" are used to indicate whether this is the first
# sequence or the second sequence. The embedding vectors for `type=0` and
# `type=1` were learned during pre-training and are added to the wordpiece
# embedding vector (and position vector). This is not *strictly* necessary
# since the [SEP] token unambigiously separates the sequences, but it makes
# it easier for the model to learn the concept of sequences.
#
# For classification tasks, the first vector (corresponding to [CLS]) is
# used as as the "sentence vector". Note that this only makes sense because
# the entire model is fine-tuned.
tokens = []
segment_ids = []
tokens.append("[CLS]")
segment_ids.append(0)
for token in tokens_a:
tokens.append(token)
segment_ids.append(0)
tokens.append("[SEP]")
segment_ids.append(0)
assert len(tokens_b) > 0
for token in tokens_b:
tokens.append(token)
segment_ids.append(1)
tokens.append("[SEP]")
segment_ids.append(1)
input_ids = tokenizer.convert_tokens_to_ids(tokens)
# The mask has 1 for real tokens and 0 for padding tokens. Only real
# tokens are attended to.
padding_mask = [1] * len(input_ids)
# Zero-pad up to the sequence length.
while len(input_ids) < max_seq_len:
input_ids.append(0)
padding_mask.append(0)
segment_ids.append(0)
lm_label_ids.append(-1)
# Convert is_next_label: Note that in Bert, is_next_labelid = 0 is used for next_sentence=true!
if sample.clear_text["is_next_label"]:
is_next_label_id = [0]
else:
is_next_label_id = [1]
assert len(input_ids) == max_seq_len
assert len(padding_mask) == max_seq_len
assert len(segment_ids) == max_seq_len
assert len(lm_label_ids) == max_seq_len
feature_dict = {
"input_ids": input_ids,
"padding_mask": padding_mask,
"segment_ids": segment_ids,
"lm_label_ids": lm_label_ids,
"label_ids": is_next_label_id,
}
return [feature_dict]