def _get_subtokens_and_subtokens_mask( query: str, tokenizer: TokenizerSpec) -> Tuple[List[str], List[int]]: """ Tokenizes input query into subtokens and creates subtokens mask. Subtokens mask is an array of the same length as subtokens array and contains zeros and ones in which. If element of mask equals 1, then corresponding subtoken in subtokens array is first subtoken in some word Args: query: a string that will be tokenized tokenizer: an instance of tokenizer Returns: subtokens: list of subtokens subtokens_mask: list of ints """ words = query.strip().split() subtokens = [] subtokens_mask = [] for j, word in enumerate(words): word_tokens = tokenizer.text_to_tokens(word) subtokens.extend(word_tokens) subtokens_mask.append(1) subtokens_mask.extend([0] * (len(word_tokens) - 1)) return subtokens, subtokens_mask
def convert_examples_to_features( self, examples: List[str], label_list: List[int], max_seq_length: int, tokenizer: TokenizerSpec, output_mode: str, bos_token: str = None, eos_token: str = '[SEP]', pad_token: str = '[PAD]', cls_token: str = '[CLS]', sep_token_extra: str = None, cls_token_at_end: bool = False, cls_token_segment_id: int = 0, pad_token_segment_id: int = 0, pad_on_left: bool = False, mask_padding_with_zero: bool = True, sequence_a_segment_id: int = 0, sequence_b_segment_id: int = 1, ): """ Loads a data file into a list of `InputBatch`s. The `cls_token_at_end` defines the location of the CLS token: * False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP] * True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS] The `cls_token_segment_id` defines the segment id associated to the CLS token (0 for BERT, 2 for XLNet) The convention in BERT is: a. For sequence pairs: * tokens: [CLS] is this jack ##ville ? [SEP] no it is not . [SEP] * type_ids: 0 0 0 0 0 0 0 1 1 1 1 1 1 b. For single sequences: * tokens: [CLS] the dog is hairy . [SEP] * type_ids: 0 0 0 0 0 0 0 Where "type_ids" are used to indicate whether this is the first sequence or the second sequence. The embedding vectors for `type=0` and `type=1` were learned during pre-training and are added to the wordpiece embedding vector (and position vector). This is not *strictly* necessarysince the [SEP] token unambiguously separates the sequences, but it makes it easier for the model to learn the concept of sequences. For classification tasks, the first vector (corresponding to [CLS]) is used as as the "sentence vector". Note that this only makes sense because the entire model is fine-tuned. The convention for NMT is: a. For sequence pairs: * tokens:<BOS> is this jack ##ville ? <EOS> <BOS> no it is not . <EOS> * type_ids:0 0 0 0 0 0 0 1 1 1 1 1 1 1 b. For single sequences: * tokens: <BOS> the dog is hairy . <EOS> * type_ids: 0 0 0 0 0 0 0 """ label_map = {label: i for i, label in enumerate(label_list)} features = [] for ex_index, example in enumerate(examples): if ex_index % 10000 == 0: logging.info("Writing example %d of %d" % (ex_index, len(examples))) tokens_a = tokenizer.text_to_tokens(example.text_a) tokens_b = None if example.text_b: tokens_b = tokenizer.text_to_tokens(example.text_b) special_tokens_count = 2 if eos_token else 0 special_tokens_count += 1 if sep_token_extra else 0 special_tokens_count += 2 if bos_token else 0 special_tokens_count += 1 if cls_token else 0 self._truncate_seq_pair(tokens_a, tokens_b, max_seq_length - special_tokens_count) else: special_tokens_count = 1 if eos_token else 0 special_tokens_count += 1 if sep_token_extra else 0 special_tokens_count += 1 if bos_token else 0 if len(tokens_a) > max_seq_length - special_tokens_count: tokens_a = tokens_a[:max_seq_length - special_tokens_count] # Add special tokens to sequence_a tokens = tokens_a if bos_token: tokens = [bos_token] + tokens if eos_token: tokens += [eos_token] segment_ids = [sequence_a_segment_id] * len(tokens) # Add sequence separator between sequences if tokens_b and sep_token_extra: tokens += [sep_token_extra] segment_ids += [sequence_a_segment_id] # Add special tokens to sequence_b if tokens_b: if bos_token: tokens += [bos_token] segment_ids += [sequence_b_segment_id] tokens += tokens_b segment_ids += [sequence_b_segment_id] * (len(tokens_b)) if eos_token: tokens += [eos_token] segment_ids += [sequence_b_segment_id] # Add classification token - for BERT models if cls_token: if cls_token_at_end: tokens += [cls_token] segment_ids += [cls_token_segment_id] else: tokens = [cls_token] + tokens segment_ids = [cls_token_segment_id] + segment_ids input_ids = tokenizer.tokens_to_ids(tokens) # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) # Zero-pad up to the sequence length. padding_length = max_seq_length - len(input_ids) pad_token_id = tokenizer.tokens_to_ids([pad_token])[0] if pad_on_left: input_ids = ([pad_token_id] * padding_length) + input_ids input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids else: input_ids = input_ids + ([pad_token_id] * padding_length) input_mask = input_mask + ( [0 if mask_padding_with_zero else 1] * padding_length) segment_ids = segment_ids + ([pad_token_segment_id] * padding_length) if len(input_ids) != max_seq_length: raise ValueError("input_ids must be of length max_seq_length") if len(input_mask) != max_seq_length: raise ValueError("input_mask must be of length max_seq_length") if len(segment_ids) != max_seq_length: raise ValueError( "segment_ids must be of length max_seq_length") if output_mode == "classification": label_id = label_map[example.label] elif output_mode == "regression": label_id = np.float32(example.label) else: raise KeyError(output_mode) if ex_index < 5: logging.info("*** Example ***") logging.info("guid: %s" % (example.guid)) logging.info("tokens: %s" % " ".join(list(map(str, tokens)))) logging.info("input_ids: %s" % " ".join(list(map(str, input_ids)))) logging.info("input_mask: %s" % " ".join(list(map(str, input_mask)))) logging.info("segment_ids: %s" % " ".join(list(map(str, segment_ids)))) logging.info("label: %s (id = %d)" % (example.label, label_id)) features.append( InputFeatures(input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids, label_id=label_id)) return features
def get_features( queries: List[str], max_seq_length: int, tokenizer: TokenizerSpec, punct_label_ids: dict = None, capit_label_ids: dict = None, pad_label: str = 'O', punct_labels_lines=None, capit_labels_lines=None, ignore_extra_tokens=False, ignore_start_end: Optional[bool] = False, ): """ Processes the data and returns features. Args: queries: text sequences max_seq_length: max sequence length minus 2 for [CLS] and [SEP] tokenizer: such as AutoTokenizer pad_label: pad value use for labels. By default, it's the neutral label. punct_label_ids: dict to map punctuation labels to label ids. Starts with pad_label->0 and then increases in alphabetical order. Required for training and evaluation, not needed for inference. capit_label_ids: dict to map labels to label ids. Starts with pad_label->0 and then increases in alphabetical order. Required for training and evaluation, not needed for inference. punct_labels: list of labels for every word in a sequence (str) capit_labels: list of labels for every word in a sequence (str) ignore_extra_tokens: whether to ignore extra tokens in the loss_mask ignore_start_end: whether to ignore bos and eos tokens in the loss_mask Returns: all_input_ids: input ids for all tokens all_segment_ids: token type ids all_input_mask: attention mask to use for BERT model all_subtokens_mask: masks out all subwords besides the first one all_loss_mask: loss mask to mask out tokens during training punct_all_labels: all labels for punctuation task (ints) capit_all_labels: all labels for capitalization task (ints) punct_label_ids: label (str) to id (int) map for punctuation task capit_label_ids: label (str) to id (int) map for capitalization task """ all_subtokens = [] all_loss_mask = [] all_subtokens_mask = [] all_segment_ids = [] all_input_ids = [] all_input_mask = [] sent_lengths = [] punct_all_labels = [] capit_all_labels = [] with_label = False if punct_labels_lines and capit_labels_lines: with_label = True for i, query in enumerate(queries): words = query.strip().split() # add bos token subtokens = [tokenizer.cls_token] loss_mask = [1 - ignore_start_end] subtokens_mask = [0] if with_label: pad_id = punct_label_ids[pad_label] punct_labels = [pad_id] punct_query_labels = [ punct_label_ids[lab] for lab in punct_labels_lines[i] ] capit_labels = [pad_id] capit_query_labels = [ capit_label_ids[lab] for lab in capit_labels_lines[i] ] for j, word in enumerate(words): word_tokens = tokenizer.text_to_tokens(word) subtokens.extend(word_tokens) loss_mask.append(1) loss_mask.extend([int(not ignore_extra_tokens)] * (len(word_tokens) - 1)) subtokens_mask.append(1) subtokens_mask.extend([0] * (len(word_tokens) - 1)) if with_label: punct_labels.extend([punct_query_labels[j]] * len(word_tokens)) capit_labels.extend([capit_query_labels[j]] * len(word_tokens)) # add eos token subtokens.append(tokenizer.sep_token) loss_mask.append(1 - ignore_start_end) subtokens_mask.append(0) sent_lengths.append(len(subtokens)) all_subtokens.append(subtokens) all_loss_mask.append(loss_mask) all_subtokens_mask.append(subtokens_mask) all_input_mask.append([1] * len(subtokens)) if with_label: punct_labels.append(pad_id) punct_all_labels.append(punct_labels) capit_labels.append(pad_id) capit_all_labels.append(capit_labels) max_seq_length = min(max_seq_length, max(sent_lengths)) logging.info(f'Max length: {max_seq_length}') get_stats(sent_lengths) too_long_count = 0 for i, subtokens in enumerate(all_subtokens): if len(subtokens) > max_seq_length: subtokens = [tokenizer.cls_token] + subtokens[-max_seq_length + 1:] all_input_mask[i] = [1] + all_input_mask[i][-max_seq_length + 1:] all_loss_mask[i] = [int(not ignore_start_end) ] + all_loss_mask[i][-max_seq_length + 1:] all_subtokens_mask[i] = [ 0 ] + all_subtokens_mask[i][-max_seq_length + 1:] if with_label: punct_all_labels[i] = [ pad_id ] + punct_all_labels[i][-max_seq_length + 1:] capit_all_labels[i] = [ pad_id ] + capit_all_labels[i][-max_seq_length + 1:] too_long_count += 1 all_input_ids.append(tokenizer.tokens_to_ids(subtokens)) if len(subtokens) < max_seq_length: extra = max_seq_length - len(subtokens) all_input_ids[i] = all_input_ids[i] + [0] * extra all_loss_mask[i] = all_loss_mask[i] + [0] * extra all_subtokens_mask[i] = all_subtokens_mask[i] + [0] * extra all_input_mask[i] = all_input_mask[i] + [0] * extra if with_label: punct_all_labels[i] = punct_all_labels[i] + [pad_id] * extra capit_all_labels[i] = capit_all_labels[i] + [pad_id] * extra all_segment_ids.append([0] * max_seq_length) logging.info(f'{too_long_count} are longer than {max_seq_length}') for i in range(min(len(all_input_ids), 5)): logging.info("*** Example ***") logging.info("i: %s" % (i)) logging.info("subtokens: %s" % " ".join(list(map(str, all_subtokens[i])))) logging.info("loss_mask: %s" % " ".join(list(map(str, all_loss_mask[i])))) logging.info("input_mask: %s" % " ".join(list(map(str, all_input_mask[i])))) logging.info("subtokens_mask: %s" % " ".join(list(map(str, all_subtokens_mask[i])))) if with_label: logging.info("punct_labels: %s" % " ".join(list(map(str, punct_all_labels[i])))) logging.info("capit_labels: %s" % " ".join(list(map(str, capit_all_labels[i])))) return ( all_input_ids, all_segment_ids, all_input_mask, all_subtokens_mask, all_loss_mask, punct_all_labels, capit_all_labels, punct_label_ids, capit_label_ids, )
def get_features( queries: List[str], tokenizer: TokenizerSpec, max_seq_length: int = -1, label_ids: dict = None, pad_label: str = 'O', raw_labels: List[str] = None, ignore_extra_tokens: bool = False, ignore_start_end: bool = False, ): """ Processes the data and returns features. Args: queries: text sequences tokenizer: such as AutoTokenizer max_seq_length: max sequence length minus 2 for [CLS] and [SEP], when -1 - use the max len from the data pad_label: pad value use for labels. By default, it's the neutral label. raw_labels: list of labels for every word in a sequence label_ids: dict to map labels to label ids. Starts with pad_label->0 and then increases in alphabetical order. Required for training and evaluation, not needed for inference. ignore_extra_tokens: whether to ignore extra tokens in the loss_mask ignore_start_end: whether to ignore bos and eos tokens in the loss_mask """ all_subtokens = [] all_loss_mask = [] all_subtokens_mask = [] all_segment_ids = [] all_input_ids = [] all_input_mask = [] sent_lengths = [] all_labels = [] with_label = False if raw_labels is not None: with_label = True for i, query in enumerate(queries): words = query.strip().split() # add bos token subtokens = [tokenizer.cls_token] loss_mask = [1 - ignore_start_end] subtokens_mask = [0] if with_label: pad_id = label_ids[pad_label] labels = [pad_id] query_labels = [label_ids[lab] for lab in raw_labels[i]] for j, word in enumerate(words): word_tokens = tokenizer.text_to_tokens(word) # to handle emojis that could be neglected during tokenization if len(word.strip()) > 0 and len(word_tokens) == 0: word_tokens = [tokenizer.ids_to_tokens(tokenizer.unk_id)] subtokens.extend(word_tokens) loss_mask.append(1) loss_mask.extend([int(not ignore_extra_tokens)] * (len(word_tokens) - 1)) subtokens_mask.append(1) subtokens_mask.extend([0] * (len(word_tokens) - 1)) if with_label: labels.extend([query_labels[j]] * len(word_tokens)) # add eos token subtokens.append(tokenizer.sep_token) loss_mask.append(1 - ignore_start_end) subtokens_mask.append(0) sent_lengths.append(len(subtokens)) all_subtokens.append(subtokens) all_loss_mask.append(loss_mask) all_subtokens_mask.append(subtokens_mask) all_input_mask.append([1] * len(subtokens)) if with_label: labels.append(pad_id) all_labels.append(labels) max_seq_length_data = max(sent_lengths) max_seq_length = min(max_seq_length, max_seq_length_data) if max_seq_length > 0 else max_seq_length_data logging.info(f'Setting Max Seq length to: {max_seq_length}') get_stats(sent_lengths) too_long_count = 0 for i, subtokens in enumerate(all_subtokens): if len(subtokens) > max_seq_length: subtokens = [tokenizer.cls_token] + subtokens[-max_seq_length + 1 :] all_input_mask[i] = [1] + all_input_mask[i][-max_seq_length + 1 :] all_loss_mask[i] = [int(not ignore_start_end)] + all_loss_mask[i][-max_seq_length + 1 :] all_subtokens_mask[i] = [0] + all_subtokens_mask[i][-max_seq_length + 1 :] if with_label: all_labels[i] = [pad_id] + all_labels[i][-max_seq_length + 1 :] too_long_count += 1 all_input_ids.append(tokenizer.tokens_to_ids(subtokens)) if len(subtokens) < max_seq_length: extra = max_seq_length - len(subtokens) all_input_ids[i] = all_input_ids[i] + [0] * extra all_loss_mask[i] = all_loss_mask[i] + [0] * extra all_subtokens_mask[i] = all_subtokens_mask[i] + [0] * extra all_input_mask[i] = all_input_mask[i] + [0] * extra if with_label: all_labels[i] = all_labels[i] + [pad_id] * extra all_segment_ids.append([0] * max_seq_length) logging.warning(f'{too_long_count} are longer than {max_seq_length}') for i in range(min(len(all_input_ids), 1)): logging.info("*** Example ***") logging.info("i: %s", i) logging.info("subtokens: %s", " ".join(list(map(str, all_subtokens[i])))) logging.info("loss_mask: %s", " ".join(list(map(str, all_loss_mask[i])))) logging.info("input_mask: %s", " ".join(list(map(str, all_input_mask[i])))) logging.info("subtokens_mask: %s", " ".join(list(map(str, all_subtokens_mask[i])))) if with_label: logging.info("labels: %s", " ".join(list(map(str, all_labels[i])))) return (all_input_ids, all_segment_ids, all_input_mask, all_subtokens_mask, all_loss_mask, all_labels)