def text_to_instance( self, # type: ignore question_text: str, passage_text: str, char_spans: List[Tuple[int, int]] = None, answer_texts: List[str] = None, passage_tokens: List[Token] = None, additional_metadata: Dict[str, Any] = None, ) -> Optional[Instance]: if not passage_tokens: passage_tokens = self._tokenizer.tokenize(passage_text) question_tokens = self._tokenizer.tokenize(question_text) if self.passage_length_limit is not None: passage_tokens = passage_tokens[:self.passage_length_limit] if self.question_length_limit is not None: question_tokens = question_tokens[:self.question_length_limit] char_spans = char_spans or [] # We need to convert character indices in `passage_text` to token indices in # `passage_tokens`, as the latter is what we'll actually use for supervision. token_spans: List[Tuple[int, int]] = [] passage_offsets = [(token.idx, token.idx + len(token.text)) for token in passage_tokens] for char_span_start, char_span_end in char_spans: if char_span_end > passage_offsets[-1][1]: continue (span_start, span_end), error = utils.char_span_to_token_span( passage_offsets, (char_span_start, char_span_end)) if error: logger.debug("Passage: %s", passage_text) logger.debug("Passage tokens: %s", passage_tokens) logger.debug("Question text: %s", question_text) logger.debug("Answer span: (%d, %d)", char_span_start, char_span_end) logger.debug("Token span: (%d, %d)", span_start, span_end) logger.debug("Tokens in answer: %s", passage_tokens[span_start:span_end + 1]) logger.debug("Answer: %s", passage_text[char_span_start:char_span_end]) token_spans.append((span_start, span_end)) # The original answer is filtered out if char_spans and not token_spans: if self.skip_invalid_examples: return None else: token_spans.append( (len(passage_tokens) - 1, len(passage_tokens) - 1)) return utils.make_reading_comprehension_instance( question_tokens, passage_tokens, self._token_indexers, passage_text, token_spans, answer_texts, additional_metadata, )
def text_to_instance( self, # type: ignore question_text: str, passage_text: str, token_spans: List[Tuple[int, int]] = None, answer_texts: List[str] = None, question_tokens: List[Token] = None, passage_tokens: List[Token] = None, ) -> Instance: if not question_tokens: question_tokens = self._tokenizer.tokenize(question_text) if not passage_tokens: passage_tokens = self._tokenizer.tokenize(passage_text) return utils.make_reading_comprehension_instance( question_tokens, passage_tokens, self._token_indexers, passage_text, token_spans, answer_texts, )
def text_to_instance( self, # type: ignore question_text: str, passage_text: str, question_id: str = None, passage_id: str = None, answer_annotations: List[Dict] = None, passage_tokens: List[Token] = None, ) -> Union[Instance, None]: if not passage_tokens: passage_tokens = self._tokenizer.tokenize(passage_text) passage_tokens = split_tokens_by_hyphen(passage_tokens) question_tokens = self._tokenizer.tokenize(question_text) question_tokens = split_tokens_by_hyphen(question_tokens) if self.passage_length_limit is not None: passage_tokens = passage_tokens[:self.passage_length_limit] if self.question_length_limit is not None: question_tokens = question_tokens[:self.question_length_limit] answer_type: str = None answer_texts: List[str] = [] if answer_annotations: # Currently we only use the first annotated answer here, but actually this doesn't affect # the training, because we only have one annotation for the train set. answer_type, answer_texts = self.extract_answer_info_from_annotation( answer_annotations[0]) # Tokenize the answer text in order to find the matched span based on token tokenized_answer_texts = [] for answer_text in answer_texts: answer_tokens = self._tokenizer.tokenize(answer_text) answer_tokens = split_tokens_by_hyphen(answer_tokens) tokenized_answer_texts.append(" ".join(token.text for token in answer_tokens)) if self.instance_format == "squad": valid_passage_spans = (self.find_valid_spans( passage_tokens, tokenized_answer_texts) if tokenized_answer_texts else []) if not valid_passage_spans: if "passage_span" in self.skip_when_all_empty: return None else: valid_passage_spans.append( (len(passage_tokens) - 1, len(passage_tokens) - 1)) return make_reading_comprehension_instance( question_tokens, passage_tokens, self._token_indexers, passage_text, valid_passage_spans, # this `answer_texts` will not be used for evaluation answer_texts, additional_metadata={ "original_passage": passage_text, "original_question": question_text, "passage_id": passage_id, "question_id": question_id, "valid_passage_spans": valid_passage_spans, "answer_annotations": answer_annotations, }, ) elif self.instance_format == "bert": question_concat_passage_tokens = question_tokens + [ Token("[SEP]") ] + passage_tokens valid_passage_spans = [] for span in self.find_valid_spans(passage_tokens, tokenized_answer_texts): # This span is for `question + [SEP] + passage`. valid_passage_spans.append( (span[0] + len(question_tokens) + 1, span[1] + len(question_tokens) + 1)) if not valid_passage_spans: if "passage_span" in self.skip_when_all_empty: return None else: valid_passage_spans.append(( len(question_concat_passage_tokens) - 1, len(question_concat_passage_tokens) - 1, )) answer_info = { "answer_texts": answer_texts, # this `answer_texts` will not be used for evaluation "answer_passage_spans": valid_passage_spans, } return self.make_bert_drop_instance( question_tokens, passage_tokens, question_concat_passage_tokens, self._token_indexers, passage_text, answer_info, additional_metadata={ "original_passage": passage_text, "original_question": question_text, "passage_id": passage_id, "question_id": question_id, "answer_annotations": answer_annotations, }, ) elif self.instance_format == "drop": numbers_in_passage = [] number_indices = [] for token_index, token in enumerate(passage_tokens): number = self.convert_word_to_number(token.text) if number is not None: numbers_in_passage.append(number) number_indices.append(token_index) # hack to guarantee minimal length of padded number numbers_in_passage.append(0) number_indices.append(-1) numbers_as_tokens = [ Token(str(number)) for number in numbers_in_passage ] valid_passage_spans = (self.find_valid_spans( passage_tokens, tokenized_answer_texts) if tokenized_answer_texts else []) valid_question_spans = (self.find_valid_spans( question_tokens, tokenized_answer_texts) if tokenized_answer_texts else []) target_numbers = [] # `answer_texts` is a list of valid answers. for answer_text in answer_texts: number = self.convert_word_to_number(answer_text) if number is not None: target_numbers.append(number) valid_signs_for_add_sub_expressions: List[List[int]] = [] valid_counts: List[int] = [] if answer_type in ["number", "date"]: valid_signs_for_add_sub_expressions = self.find_valid_add_sub_expressions( numbers_in_passage, target_numbers) if answer_type in ["number"]: # Currently we only support count number 0 ~ 9 numbers_for_count = list(range(10)) valid_counts = self.find_valid_counts(numbers_for_count, target_numbers) type_to_answer_map = { "passage_span": valid_passage_spans, "question_span": valid_question_spans, "addition_subtraction": valid_signs_for_add_sub_expressions, "counting": valid_counts, } if self.skip_when_all_empty and not any( type_to_answer_map[skip_type] for skip_type in self.skip_when_all_empty): return None answer_info = { "answer_texts": answer_texts, # this `answer_texts` will not be used for evaluation "answer_passage_spans": valid_passage_spans, "answer_question_spans": valid_question_spans, "signs_for_add_sub_expressions": valid_signs_for_add_sub_expressions, "counts": valid_counts, } return self.make_marginal_drop_instance( question_tokens, passage_tokens, numbers_as_tokens, number_indices, self._token_indexers, passage_text, answer_info, additional_metadata={ "original_passage": passage_text, "original_question": question_text, "original_numbers": numbers_in_passage, "passage_id": passage_id, "question_id": question_id, "answer_info": answer_info, "answer_annotations": answer_annotations, }, ) else: raise ValueError( f'Expect the instance format to be "drop", "squad" or "bert", ' f"but got {self.instance_format}")
def text_to_instance( self, # type: ignore question_text: str, passage_text: str, is_impossible: bool = None, char_spans: List[Tuple[int, int]] = None, answer_texts: List[str] = None, passage_tokens: List[Token] = None, additional_metadata: Dict[str, Any] = None, ) -> Optional[Instance]: if not passage_tokens: passage_tokens = self._tokenizer.tokenize(passage_text) if self.no_answer_token is not None: if self.passage_length_limit is not None: passage_tokens = passage_tokens[:self.passage_length_limit - 1] passage_tokens = passage_tokens + [ Token( text=self.no_answer_token, idx=passage_tokens[-1].idx + len(passage_tokens[-1].text) + 1, # type: ignore lemma_=self.no_answer_token, ) ] elif self.passage_length_limit is not None: passage_tokens = passage_tokens[:self.passage_length_limit] question_tokens = self._tokenizer.tokenize(question_text) if self.question_length_limit is not None: question_tokens = question_tokens[:self.question_length_limit] if is_impossible: if self.no_answer_token is None: raise ValueError( "This is a SQuAD 2.0 dataset, yet your using a SQuAD reader has 'no_answer_token' " "set to `None`. " "Consider specifying the 'no_answer_token' or using the 'squad2' reader instead, which " f"by default uses '{SQUAD2_NO_ANSWER_TOKEN}' as the 'no_answer_token'." ) answer_texts = [self.no_answer_token] token_spans: List[Tuple[int, int]] = [(len(passage_tokens) - 1, len(passage_tokens) - 1)] else: char_spans = char_spans or [] # We need to convert character indices in `passage_text` to token indices in # `passage_tokens`, as the latter is what we'll actually use for supervision. token_spans = [] passage_offsets = [(token.idx, token.idx + len(token.text)) for token in passage_tokens] for char_span_start, char_span_end in char_spans: if char_span_end > passage_offsets[-1][1]: continue (span_start, span_end), error = utils.char_span_to_token_span( passage_offsets, (char_span_start, char_span_end)) if error: logger.debug("Passage: %s", passage_text) logger.debug("Passage tokens (with no-answer): %s", passage_tokens) logger.debug("Question text: %s", question_text) logger.debug("Answer span: (%d, %d)", char_span_start, char_span_end) logger.debug("Token span: (%d, %d)", span_start, span_end) logger.debug( "Tokens in answer: %s", passage_tokens[span_start:span_end + 1], ) logger.debug("Answer: %s", passage_text[char_span_start:char_span_end]) token_spans.append((span_start, span_end)) # The original answer is filtered out if char_spans and not token_spans: if self.skip_impossible_questions: return None else: if self.no_answer_token is not None: answer_texts = [self.no_answer_token] token_spans.append(( len(passage_tokens) - 1, len(passage_tokens) - 1, )) return utils.make_reading_comprehension_instance( question_tokens, passage_tokens, self._token_indexers, passage_text, token_spans, answer_texts, additional_metadata, )