def test_random_sampling(self, seed): interaction = _to_interaction( [[1.0, 3.0, 6.0], [2.0, 0.0, None], [1.0, None, 0.0]], 3.0) rng = random.Random(seed) interpretation_utils._MAX_NUM_CANDIDATES = 1 actual = interpretation_utils.find_candidates(rng, interaction.table, interaction.questions[0]) self.assertLen(actual, 3)
def test_selection_answer(self, seed): interaction = _to_interaction( [[1.0, 3.0, 6.0], [2.0, 0.0, None], [1.0, None, 0.0]], 100.0) coords = interaction.questions[0].answer.answer_coordinates.add() coords.row_index = 1 coords.column_index = 2 rng = random.Random(seed) actual = interpretation_utils.find_candidates(rng, interaction.table, interaction.questions[0]) expected = [_Candidate(_AggFun.NONE, 2, (1, ))] self.assertEqual(expected, actual)
def test_random_exploration(self, seed): interaction = _to_interaction( [[1.0, 3.0, 6.0], [2.0, 0.0, None], [1.0, None, 0.0]], 3.0) rng = random.Random(seed) interpretation_utils._MAX_INDICES_TO_EXPLORE = 1 actual = interpretation_utils.find_candidates(rng, interaction.table, interaction.questions[0]) expected = [ _Candidate(_AggFun.COUNT, 0, (0, 1, 2)), _Candidate(_AggFun.COUNT, 1, (0, 1, 2)), _Candidate(_AggFun.COUNT, 2, (0, 1, 2)), _Candidate(_AggFun.SUM, 1, (0, )), _Candidate(_AggFun.AVERAGE, 1, (0, )), ] self.assertEqual(expected, actual)
def test_float_conversion(self, seed): interaction = _to_interaction( [[1.0, 3.0, 6.0], [2.0, 0.0, None], [1.0, None, 0.0]], 3.0) rng = random.Random(seed) actual = interpretation_utils.find_candidates(rng, interaction.table, interaction.questions[0]) expected = [ _Candidate(_AggFun.COUNT, 0, (0, 1, 2)), _Candidate(_AggFun.COUNT, 1, (0, 1, 2)), _Candidate(_AggFun.COUNT, 2, (0, 1, 2)), _Candidate(_AggFun.SUM, 0, (0, 1)), _Candidate(_AggFun.SUM, 0, (1, 2)), _Candidate(_AggFun.SUM, 1, (0, )), _Candidate(_AggFun.SUM, 1, (0, 1)), _Candidate(_AggFun.AVERAGE, 1, (0, )), _Candidate(_AggFun.AVERAGE, 2, (0, 2)), ] self.assertEqual(expected, actual)
def convert(self, interaction, index): """Converts question at 'index' to example.""" table = interaction.table num_rows = self._get_num_rows(table, self._drop_rows_to_fit) num_columns = self._get_num_columns(table) question = interaction.questions[index] if not interaction.questions[index].answer.is_valid: raise ValueError('Invalid answer') text_tokens = self._tokenize_extended_question(question, table) tokenized_table = self._tokenize_table(table) serialized_example, features = self._to_trimmed_features( question=question, table=table, question_tokens=text_tokens, tokenized_table=tokenized_table, num_columns=num_columns, num_rows=num_rows, drop_rows_to_fit=self._drop_rows_to_fit) column_ids = serialized_example.column_ids row_ids = serialized_example.row_ids def get_answer_ids(question): if self._update_answer_coordinates: return _find_answer_ids_from_answer_texts( column_ids, row_ids, tokenized_table, answer_texts=[ self._tokenizer.tokenize(at) for at in question.answer.answer_texts ], ) return _get_answer_ids(column_ids, row_ids, question) answer_ids = get_answer_ids(question) self._pad_to_seq_length(answer_ids) features['label_ids'] = create_int_feature(answer_ids) if index == 0: prev_answer_ids = [0] * len(column_ids) else: prev_answer_ids = get_answer_ids(interaction.questions[index - 1],) self._pad_to_seq_length(prev_answer_ids) features['prev_label_ids'] = create_int_feature(prev_answer_ids) features['question_id'] = create_string_feature( [question.id.encode('utf8')]) features['question_id_ints'] = create_int_feature( text_utils.str_to_ints( question.id, length=text_utils.DEFAULT_INTS_LENGTH)) features['aggregation_function_id'] = create_int_feature( [question.answer.aggregation_function]) features['classification_class_index'] = create_int_feature( [question.answer.class_index]) answer = question.answer.float_value if question.answer.HasField( 'float_value') else _NAN features['answer'] = create_float_feature([answer]) self._add_question_numeric_values(question, features) if self._add_aggregation_candidates: rng = random.Random(fingerprint(question.id)) candidates = interpretation_utils.find_candidates( rng, table, question) num_initial_candidates = len(candidates) candidates = [c for c in candidates if len(c.rows) < _MAX_NUM_ROWS] candidates = candidates[:_MAX_NUM_CANDIDATES] funs = [0] * _MAX_NUM_CANDIDATES sizes = [0] * _MAX_NUM_CANDIDATES indexes = [] num_final_candidates = 0 for index, candidate in enumerate(candidates): token_indexes = [] for row in candidate.rows: token_indexes += _get_cell_token_indexes(column_ids, row_ids, candidate.column, row) if len(indexes) + len(serialized_example.tokens) > _MAX_INDEX_LENGTH: break num_final_candidates += 1 sizes[index] = len(token_indexes) funs[index] = candidate.agg_function indexes += token_indexes # <int>[1] features['cand_num'] = create_int_feature([num_final_candidates]) # <int>[_MAX_NUM_CANDIDATES] features['can_aggregation_function_ids'] = create_int_feature(funs) # <int>[_MAX_NUM_CANDIDATES] features['can_sizes'] = create_int_feature(sizes) # <int>[_MAX_INDEX_LENGTH] # Actual length is sum(sizes). features['can_indexes'] = create_int_feature(indexes) return tf.train.Example(features=tf.train.Features(feature=features))
def convert(self, interaction, index): """Converts question at 'index' to example.""" table = interaction.table num_rows = self._get_num_rows(table, self._drop_rows_to_fit) num_columns = self._get_num_columns(table) question = interaction.questions[index] if not interaction.questions[index].answer.is_valid: beam_metrics.Metrics.counter( _NS, 'Conversion skipped (answer not valid)').inc() raise ValueError('Invalid answer') text_tokens = self._tokenize_extended_question(question, table) tokenized_table = self._tokenize_table(table) table_selection_ext = table_selection_pb2.TableSelection.table_selection_ext if table_selection_ext in question.Extensions: table_selection = question.Extensions[table_selection_ext] if not tokenized_table.selected_tokens: raise ValueError('No tokens selected') if table_selection.selected_tokens: selected_tokens = {(t.row_index, t.column_index, t.token_index) for t in table_selection.selected_tokens} tokenized_table.selected_tokens = [ t for t in tokenized_table.selected_tokens if (t.row_index, t.column_index, t.token_index) in selected_tokens ] serialized_example, features = self._to_trimmed_features( question=question, table=table, question_tokens=text_tokens, tokenized_table=tokenized_table, num_columns=num_columns, num_rows=num_rows, drop_rows_to_fit=self._drop_rows_to_fit) column_ids = serialized_example.column_ids row_ids = serialized_example.row_ids def get_answer_ids(question): if self._update_answer_coordinates: return _find_answer_ids_from_answer_texts( column_ids, row_ids, tokenized_table, answer_texts=[ self._tokenizer.tokenize(at) for at in question.answer.answer_texts ], ) return _get_answer_ids(column_ids, row_ids, question) answer_ids = get_answer_ids(question) self._pad_to_seq_length(answer_ids) features['label_ids'] = create_int_feature(answer_ids) if index > 0: prev_answer_ids = get_answer_ids(interaction.questions[index - 1],) else: prev_answer_ids = [0] * len(column_ids) self._pad_to_seq_length(prev_answer_ids) features['prev_label_ids'] = create_int_feature(prev_answer_ids) features['question_id'] = create_string_feature( [question.id.encode('utf8')]) if self._trim_question_ids: question_id = question.id[-text_utils.DEFAULT_INTS_LENGTH:] else: question_id = question.id features['question_id_ints'] = create_int_feature( text_utils.str_to_ints( question_id, length=text_utils.DEFAULT_INTS_LENGTH)) features['aggregation_function_id'] = create_int_feature( [question.answer.aggregation_function]) features['classification_class_index'] = create_int_feature( [question.answer.class_index]) answer = question.answer.float_value if question.answer.HasField( 'float_value') else _NAN features['answer'] = create_float_feature([answer]) if self._add_aggregation_candidates: rng = random.Random(fingerprint(question.id)) candidates = interpretation_utils.find_candidates(rng, table, question) num_initial_candidates = len(candidates) candidates = [c for c in candidates if len(c.rows) < _MAX_NUM_ROWS] candidates = candidates[:_MAX_NUM_CANDIDATES] funs = [0] * _MAX_NUM_CANDIDATES sizes = [0] * _MAX_NUM_CANDIDATES indexes = [] num_final_candidates = 0 for index, candidate in enumerate(candidates): token_indexes = [] for row in candidate.rows: token_indexes += _get_cell_token_indexes(column_ids, row_ids, candidate.column, row) if len(indexes) + len(serialized_example.tokens) > _MAX_INDEX_LENGTH: break num_final_candidates += 1 sizes[index] = len(token_indexes) funs[index] = candidate.agg_function indexes += token_indexes # <int>[1] features['cand_num'] = create_int_feature([num_final_candidates]) # <int>[_MAX_NUM_CANDIDATES] features['can_aggregation_function_ids'] = create_int_feature(funs) # <int>[_MAX_NUM_CANDIDATES] features['can_sizes'] = create_int_feature(sizes) # <int>[_MAX_INDEX_LENGTH] # Actual length is sum(sizes). features['can_indexes'] = create_int_feature(indexes) if num_initial_candidates > 0: beam_metrics.Metrics.counter( _NS, _get_buckets(num_initial_candidates, [10, 20, 50, 100, 200, 500, 1000, 1200, 1500], 'Candidates Size:')).inc() beam_metrics.Metrics.counter(_NS, 'Candidates: Input').inc() if num_final_candidates != num_initial_candidates: beam_metrics.Metrics.counter(_NS, 'Candidates: Dropped candidates').inc() return tf.train.Example(features=tf.train.Features(feature=features))