def process(self, question_answer_evidence): metrics.Metrics.counter(METRICS_NAMESPACE, 'num_questions').inc() if self.generate_answers: oracle_answers = [] for answer in question_answer_evidence.answer.values: oracle_answers.extend( self.extractive_oracle.find_approximate_answers( question_answer_evidence.evidence.text, answer, remove_all_stopwords_answers=True)) metrics.Metrics.distribution(METRICS_NAMESPACE, 'oracle_answers_per_question').update( len(oracle_answers)) answer_set = question_answer_evidence.answer.make_answer_set( oracle_answers) normalized_answer_set = { normalize_answer(answer) for answer in answer_set } sentences = [] for sentence in self._split_into_sentences( question_answer_evidence.evidence): sentence_obj = self._annotate_entities(sentence) metrics.Metrics.counter(METRICS_NAMESPACE, 'nltk_entities').inc( sentence_obj.num_annotations(1)) if self.generate_answers: annotations = find_answer_annotations(sentence_obj.text, answer_set) sentence_obj.annotations.extend(annotations) sentences.append(sentence_obj) big_document = data_utils.BertDocument( sentences=sentences, document_id=question_answer_evidence.question.id) metrics.Metrics.distribution(METRICS_NAMESPACE, 'num_sentences_per_question').update( len(sentences)) metrics.Metrics.distribution(METRICS_NAMESPACE, 'doc_length_per_question').update( big_document.num_characters()) if self.generate_answers: num_annotations = big_document.num_annotations(0) metrics.Metrics.distribution( METRICS_NAMESPACE, 'num_annotations_per_question').update(num_annotations) if num_annotations == 0: metrics.Metrics.counter( METRICS_NAMESPACE, 'make_example_status.answer_span_not_found').inc() yield beam.pvalue.TaggedOutput( MakeExampleOutput.NO_ANSWER, question_answer_evidence.to_json()) return tokenized_big_document = data_utils.tokenize_document_for_bert( big_document, self.tokenizer) metrics.Metrics.distribution( METRICS_NAMESPACE, 'tokenized_doc_length_per_question').update( tokenized_big_document.num_tokens()) tokenized_question = self._tokenize_text( question_answer_evidence.question.value) metrics.Metrics.distribution(METRICS_NAMESPACE, 'question_length').update( len(tokenized_question)) filtered_annotations = [] if self.generate_answers: for i, sentence in enumerate(tokenized_big_document.sentences): should_update, annotations, current_filtered_annotations = self._verify_annotations( sentence.annotations, normalized_answer_set) if should_update: tokenized_big_document.sentences[ i].annotations = annotations # pylint: disable=g-complex-comprehension filtered_annotations.extend([ FilteredAnnotation( question=question_answer_evidence.question, answer=question_answer_evidence.answer, annotation=annotation, sentence=''.join(sentence.tokens)) for annotation in current_filtered_annotations ]) metrics.Metrics.counter( METRICS_NAMESPACE, 'num_filtered_annotations').inc( len(current_filtered_annotations)) tokenized_big_document = data_utils.split_tokenized_sentences( tokenized_big_document, max_tokens=self._get_max_tokens_per_raw_doc( len(tokenized_question)), min_tokens_for_graceful_split=math.ceil( self._get_max_tokens_per_raw_doc(len(tokenized_question)) * 0.5)) if self.generate_answers: num_annotations = tokenized_big_document.num_annotations(0) metrics.Metrics.distribution( METRICS_NAMESPACE, 'num_annotations_tokenized_per_question').update( num_annotations) if num_annotations == 0: metrics.Metrics.counter( METRICS_NAMESPACE, 'make_example_status.answer_not_found_tokenized').inc() yield beam.pvalue.TaggedOutput( MakeExampleOutput.NO_ANSWER_TOKENIZED, question_answer_evidence.to_json()) yield beam.pvalue.TaggedOutput( MakeExampleOutput.NO_ANSWER_TOKENIZED_FILTERED_ANNOTATIONS, filtered_annotations) return else: approx_num_blocks = ( tokenized_big_document.num_tokens() / (self.block_length - self.block_overlap_length - len(tokenized_question))) if (num_annotations > self.max_num_annotations_per_block * approx_num_blocks): metrics.Metrics.counter( METRICS_NAMESPACE, 'num_questions_with_too_many_answers').inc() yield beam.pvalue.TaggedOutput( MakeExampleOutput.TOO_MANY_ANSWERS, question_answer_evidence.to_json()) yield beam.pvalue.TaggedOutput( MakeExampleOutput.SUCCESS_FILTERED_ANNOTATIONS, filtered_annotations) # message = question_answer_evidence.evidence.info.id tokenized_documents = data_utils.split_tokenized_documents( tokenized_big_document, max_tokens=self._get_max_tokens_per_raw_doc( len(tokenized_question)), max_sentences=None) metrics.Metrics.distribution(METRICS_NAMESPACE, 'num_examples_per_question').update( len(tokenized_documents)) if len(tokenized_documents) > 1: metrics.Metrics.counter(METRICS_NAMESPACE, 'num_too_large_evidence').inc() if self.generate_summaries: tokenized_summary = self._tokenize_text( question_answer_evidence.evidence.summary) if len(tokenized_summary) < self.block_length: tokenized_summary.extend( [self.padding_token_id] * (self.block_length - len(tokenized_summary))) for tokenized_document in tokenized_documents: if self.generate_answers and tokenized_document.num_annotations( 0) == 0: metrics.Metrics.counter( METRICS_NAMESPACE, 'make_example_status.answer_not_found_splitted').inc() continue metrics.Metrics.counter(METRICS_NAMESPACE, 'num_examples').inc() tf_example = tokenized_document.to_tf_strided_large_example( overlap_length=self.block_overlap_length, block_length=self.block_length, padding_token_id=self.padding_token_id, prefix_token_ids=tokenized_question, max_num_annotations=self.max_num_annotations_per_block) if self.generate_summaries: num_blocks = len( tf_example.features.feature['block_ids'].int64_list.value) tf_example.features.feature[ 'summary_token_ids'].int64_list.value.extend( tokenized_summary * num_blocks) yield tf_example metrics.Metrics.counter(METRICS_NAMESPACE, 'make_example_status.success').inc()
def process(self, element): yield data_utils.tokenize_document_for_bert(element, self.tokenizer)
def process(self, question_answer_evidence): metrics.Metrics.counter(METRICS_NAMESPACE, 'num_questions').inc() if self.generate_answers: oracle_answers = [] answer_set = question_answer_evidence.answer.make_answer_set( oracle_answers) normalized_answer_set = { evaluation.normalize_answer(answer) for answer in answer_set } tokenized_question = self._tokenize_text( question_answer_evidence.question.value) metrics.Metrics.distribution(METRICS_NAMESPACE, 'question_length').update( len(tokenized_question)) filtered_annotations = [] tf_examples = [] num_answer_annotations = 0 num_answer_annotations_tokenized = 0 num_entity_annotations = 0 num_entity_annotations_tokenized = 0 no_answer, yes_answer, yes_no_answer = False, False, False if question_answer_evidence.answer.values[0] == 'yes': metrics.Metrics.counter(METRICS_NAMESPACE, 'num_answer_type.yes').inc() yes_no_answer = True yes_answer = True if question_answer_evidence.answer.values[0] == 'no': metrics.Metrics.counter(METRICS_NAMESPACE, 'num_answer_type.no').inc() yes_no_answer = True no_answer = True if yes_no_answer: metrics.Metrics.counter(METRICS_NAMESPACE, 'num_answer_type.yes_no').inc() else: metrics.Metrics.counter(METRICS_NAMESPACE, 'num_answer_type.span').inc() for evidence in question_answer_evidence.evidence: sentence = self._split_into_sentences(evidence) sentence_obj = self._annotate_entities(sentence) metrics.Metrics.counter(METRICS_NAMESPACE, 'nltk_entities').inc( sentence_obj.num_annotations(1)) if self.generate_answers and not yes_no_answer: annotations = find_answer_annotations(sentence_obj.text, answer_set) sentence_obj.annotations.extend(annotations) document = data_utils.BertDocument( sentences=[sentence_obj], document_id=question_answer_evidence.question.id) num_entity_annotations += document.num_annotations(1) num_answer_annotations += document.num_annotations(0) tokenized_document = data_utils.tokenize_document_for_bert( document, self.tokenizer) metrics.Metrics.distribution( METRICS_NAMESPACE, 'tokenized_doc_length_per_paragraph').update( tokenized_document.num_tokens()) if self.generate_answers and not yes_no_answer: assert len(tokenized_document.sentences) == 1 (should_update, annotations, current_filtered_annotations) = self._verify_annotations( tokenized_document.sentences[0].annotations, normalized_answer_set) if should_update: tokenized_document.sentences[0].annotations = annotations # pylint: disable=g-complex-comprehension filtered_annotations.extend([ FilteredAnnotation( question=question_answer_evidence.question, answer=question_answer_evidence.answer, annotation=annotation, sentence=''.join( tokenized_document.sentences[0].tokens)) for annotation in current_filtered_annotations ]) metrics.Metrics.counter( METRICS_NAMESPACE, 'num_filtered_annotations').inc( len(current_filtered_annotations)) num_entity_annotations_tokenized += tokenized_document.num_annotations( 1) num_answer_annotations_tokenized += tokenized_document.num_annotations( 0) tf_example = tokenized_document.to_tf_strided_large_example( overlap_length=self.block_overlap_length, block_length=self.block_length, padding_token_id=self.padding_token_id, prefix_token_ids=tokenized_question, max_num_annotations=self.max_num_annotations_per_block) if yes_answer: assert yes_no_answer assert not no_answer tf_example.features.feature[ 'answer_type'].int64_list.value[:] = [1] elif no_answer: assert yes_no_answer assert not yes_answer tf_example.features.feature[ 'answer_type'].int64_list.value[:] = [2] else: assert not yes_no_answer tf_example.features.feature[ 'answer_type'].int64_list.value[:] = [0] if evidence.is_supporting_fact: tf_example.features.feature[ 'is_supporting_fact'].int64_list.value[:] = [1] else: tf_example.features.feature[ 'is_supporting_fact'].int64_list.value[:] = [0] tf_examples.append(tf_example) metrics.Metrics.distribution(METRICS_NAMESPACE, 'num_paragraphs_per_question').update( len(tf_examples)) metrics.Metrics.distribution( METRICS_NAMESPACE, 'num_answer_annotations_per_question').update( num_answer_annotations) metrics.Metrics.distribution( METRICS_NAMESPACE, 'num_entity_annotations_per_question').update( num_entity_annotations) if (self.generate_answers and not yes_no_answer and num_answer_annotations == 0): metrics.Metrics.counter(METRICS_NAMESPACE, 'make_example_status.no_answer').inc() yield beam.pvalue.TaggedOutput(MakeExampleOutput.NO_ANSWER, question_answer_evidence.to_json()) return metrics.Metrics.distribution( METRICS_NAMESPACE, 'num_answer_tokenize_annotations_per_question').update( num_answer_annotations_tokenized) metrics.Metrics.distribution( METRICS_NAMESPACE, 'num_entity_tokenize_annotations_per_question').update( num_entity_annotations_tokenized) metrics.Metrics.distribution(METRICS_NAMESPACE, 'num_filtered_annotations').update( len(filtered_annotations)) if (self.generate_answers and not yes_no_answer and num_answer_annotations_tokenized == 0): metrics.Metrics.counter( METRICS_NAMESPACE, 'make_example_status.no_answer_tokenized_annotations').inc() yield beam.pvalue.TaggedOutput( MakeExampleOutput.NO_ANSWER_TOKENIZED_FILTERED_ANNOTATIONS, filtered_annotations) return yield beam.pvalue.TaggedOutput( MakeExampleOutput.SUCCESS_FILTERED_ANNOTATIONS, filtered_annotations) if len(tf_examples) != 10: metrics.Metrics.counter( METRICS_NAMESPACE, 'num_not_10_paragraphs_per_question').inc() tf_example = tf_examples[0] for i in range(1, len(tf_examples)): for name in tf_example.features.feature: repeated_values = get_repeated_values(name, tf_example) extension_values = list( get_repeated_values(name, tf_examples[i])) repeated_values.extend(extension_values) metrics.Metrics.counter(METRICS_NAMESPACE, 'make_example_status.success').inc() yield tf_example
def process(self, question_answer_evidence): metrics.Metrics.counter(METRICS_NAMESPACE, 'num_questions').inc() if self.generate_answers: answer_set = question_answer_evidence.answer.make_answer_set() sentences = [] for sentence in self._split_into_sentences( question_answer_evidence.evidence): sentence_obj = self._annotate_entities(sentence) metrics.Metrics.counter(METRICS_NAMESPACE, 'nltk_entities').inc( sentence_obj.num_annotations(1)) if self.generate_answers: annotations = find_answer_annotations(sentence_obj.text, answer_set) sentence_obj.annotations.extend(annotations) sentences.append(sentence_obj) big_document = data_utils.BertDocument( sentences=sentences, document_id=question_answer_evidence.question.id) metrics.Metrics.distribution(METRICS_NAMESPACE, 'doc_length_per_question').update( big_document.num_characters()) if self.generate_answers: num_annotations = big_document.num_annotations(0) metrics.Metrics.distribution( METRICS_NAMESPACE, 'num_annotations_per_question').update(num_annotations) if num_annotations == 0: metrics.Metrics.counter( METRICS_NAMESPACE, 'make_example_status.answer_span_not_found').inc() yield beam.pvalue.TaggedOutput( MakeExampleOutput.NO_ANSWER, question_answer_evidence.to_json()) return tokenized_big_document = data_utils.tokenize_document_for_bert( big_document, self.tokenizer) metrics.Metrics.distribution( METRICS_NAMESPACE, 'tokenized_doc_length_per_question').update( tokenized_big_document.num_tokens()) tokenized_question = self._tokenize_question( question_answer_evidence.question.value) metrics.Metrics.distribution(METRICS_NAMESPACE, 'question_length').update( len(tokenized_question)) filtered_annotations = [] if self.generate_answers: for i, sentence in enumerate(tokenized_big_document.sentences): (should_update, annotations, current_filtered_annotations) = self._verify_annotations( sentence.annotations, answer_set) if should_update: tokenized_big_document.sentences[ i].annotations = annotations # pylint: disable=g-complex-comprehension filtered_annotations.extend([ FilteredAnnotation( question=question_answer_evidence.question, answer=question_answer_evidence.answer, annotation=annotation, sentence=''.join(sentence.tokens)) for annotation in current_filtered_annotations ]) metrics.Metrics.counter( METRICS_NAMESPACE, 'num_filtered_annotations').inc( len(current_filtered_annotations)) num_annotations = tokenized_big_document.num_annotations(0) metrics.Metrics.distribution( METRICS_NAMESPACE, 'num_annotations_tokenized_per_question').update( num_annotations) if num_annotations == 0: metrics.Metrics.counter( METRICS_NAMESPACE, 'make_example_status.answer_not_found_tokenized').inc() yield beam.pvalue.TaggedOutput( MakeExampleOutput.NO_ANSWER_TOKENIZED, question_answer_evidence.to_json()) yield beam.pvalue.TaggedOutput( MakeExampleOutput.NO_ANSWER_TOKENIZED_FILTERED_ANNOTATIONS, filtered_annotations) return else: approx_num_blocks = ( tokenized_big_document.num_tokens() / (self.block_length - self.block_overlap_length - len(tokenized_question))) if num_annotations > self.max_num_annotations_per_block * approx_num_blocks: metrics.Metrics.counter( METRICS_NAMESPACE, 'num_questions_with_too_many_answers').inc() yield beam.pvalue.TaggedOutput( MakeExampleOutput.TOO_MANY_ANSWERS, question_answer_evidence.to_json()) yield beam.pvalue.TaggedOutput( MakeExampleOutput.SUCCESS_FILTERED_ANNOTATIONS, filtered_annotations) tokenized_documents = data_utils.split_tokenized_documents( tokenized_big_document, max_tokens=self._get_max_tokens_per_raw_doc( len(tokenized_question)), max_sentences=None) metrics.Metrics.distribution(METRICS_NAMESPACE, 'num_examples_per_question').update( len(tokenized_documents)) if len(tokenized_documents) > 1: metrics.Metrics.counter(METRICS_NAMESPACE, 'num_too_large_evidence').inc() for tokenized_document in tokenized_documents: if self.generate_answers and tokenized_document.num_annotations( 0) == 0: metrics.Metrics.counter( METRICS_NAMESPACE, 'make_example_status.answer_not_found_splitted').inc() continue metrics.Metrics.counter(METRICS_NAMESPACE, 'num_examples').inc() yield tokenized_document.to_tf_strided_large_example( overlap_length=self.block_overlap_length, block_length=self.block_length, padding_token_id=self.padding_token_id, prefix_token_ids=tokenized_question, max_num_annotations=self.max_num_annotations_per_block) metrics.Metrics.counter(METRICS_NAMESPACE, 'make_example_status.success').inc()