def text_to_instance(self, # type: ignore
logical_forms: List[str],
table_lines: List[List[str]],
question: str) -> Instance:
# pylint: disable=arguments-differ
tokenized_question = self._tokenizer.tokenize(question.lower())
tokenized_question.insert(0, Token(START_SYMBOL))
tokenized_question.append(Token(END_SYMBOL))
question_field = TextField(tokenized_question, self._question_token_indexers)
table_context = TableQuestionContext.read_from_lines(table_lines, tokenized_question)
world = WikiTablesLanguage(table_context)
action_sequences_list: List[List[str]] = []
action_sequence_fields_list: List[TextField] = []
for logical_form in logical_forms:
try:
action_sequence = world.logical_form_to_action_sequence(logical_form)
action_sequence = reader_utils.make_bottom_up_action_sequence(action_sequence,
world.is_nonterminal)
action_sequence_field = TextField([Token(rule) for rule in action_sequence],
self._rule_indexers)
action_sequences_list.append(action_sequence)
action_sequence_fields_list.append(action_sequence_field)
except ParsingError as error:
logger.debug(f'Parsing error: {error.message}, skipping logical form')
logger.debug(f'Question was: {question}')
logger.debug(f'Logical form was: {logical_form}')
logger.debug(f'Table info was: {table_lines}')
except:
logger.error(logical_form)
raise
if not action_sequences_list:
return None
all_production_rule_fields: List[List[Field]] = []
for action_sequence in action_sequences_list:
all_production_rule_fields.append([])
for production_rule in action_sequence:
_, rule_right_side = production_rule.split(' -> ')
is_global_rule = not world.is_instance_specific_entity(rule_right_side)
field = ProductionRuleField(production_rule, is_global_rule=is_global_rule)
all_production_rule_fields[-1].append(field)
action_field = ListField([ListField(production_rule_fields) for production_rule_fields in
all_production_rule_fields])
fields = {'action_sequences': ListField(action_sequence_fields_list),
'target_tokens': question_field,
'world': MetadataField(world),
'actions': action_field}
return Instance(fields)
def text_to_instance(
self, # type: ignore
question: str,
table_lines: List[List[str]],
target_values: List[str] = None,
offline_search_output: List[str] = None) -> Instance:
"""
Reads text inputs and makes an instance. We pass the ``table_lines`` to ``TableQuestionContext``, and that
method accepts this field either as lines from CoreNLP processed tagged files that come with the dataset,
or simply in a tsv format where each line corresponds to a row and the cells are tab-separated.
Parameters
----------
question : ``str``
Input question
table_lines : ``List[List[str]]``
The table content optionally preprocessed by CoreNLP. See ``TableQuestionContext.read_from_lines``
for the expected format.
target_values : ``List[str]``, optional
Target values for the denotations the logical forms should execute to. Not required for testing.
offline_search_output : ``List[str]``, optional
List of logical forms, produced by offline search. Not required during test.
"""
# pylint: disable=arguments-differ
tokenized_question = self._tokenizer.tokenize(question.lower())
question_field = TextField(tokenized_question,
self._question_token_indexers)
metadata: Dict[str, Any] = {
"question_tokens": [x.text for x in tokenized_question]
}
table_context = TableQuestionContext.read_from_lines(
table_lines, tokenized_question)
target_values_field = MetadataField(target_values)
world = WikiTablesLanguage(table_context)
world_field = MetadataField(world)
# Note: Not passing any featre extractors when instantiating the field below. This will make
# it use all the available extractors.
table_field = KnowledgeGraphField(
table_context.get_table_knowledge_graph(),
tokenized_question,
self._table_token_indexers,
tokenizer=self._tokenizer,
include_in_vocab=self._use_table_for_vocab,
max_table_tokens=self._max_table_tokens)
production_rule_fields: List[Field] = []
for production_rule in world.all_possible_productions():
_, rule_right_side = production_rule.split(' -> ')
is_global_rule = not world.is_instance_specific_entity(
rule_right_side)
field = ProductionRuleField(production_rule,
is_global_rule=is_global_rule)
production_rule_fields.append(field)
action_field = ListField(production_rule_fields)
fields = {
'question': question_field,
'metadata': MetadataField(metadata),
'table': table_field,
'world': world_field,
'actions': action_field,
'target_values': target_values_field
}
# We'll make each target action sequence a List[IndexField], where the index is into
# the action list we made above. We need to ignore the type here because mypy doesn't
# like `action.rule` - it's hard to tell mypy that the ListField is made up of
# ProductionRuleFields.
action_map = {
action.rule: i
for i, action in enumerate(action_field.field_list)
} # type: ignore
if offline_search_output:
action_sequence_fields: List[Field] = []
for logical_form in offline_search_output:
try:
action_sequence = world.logical_form_to_action_sequence(
logical_form)
index_fields: List[Field] = []
for production_rule in action_sequence:
index_fields.append(
IndexField(action_map[production_rule],
action_field))
action_sequence_fields.append(ListField(index_fields))
except ParsingError as error:
logger.debug(
f'Parsing error: {error.message}, skipping logical form'
)
logger.debug(f'Question was: {question}')
logger.debug(f'Logical form was: {logical_form}')
logger.debug(f'Table info was: {table_lines}')
continue
except KeyError as error:
logger.debug(
f'Missing production rule: {error.args}, skipping logical form'
)
logger.debug(f'Question was: {question}')
logger.debug(f'Table info was: {table_lines}')
logger.debug(f'Logical form was: {logical_form}')
continue
except:
logger.error(logical_form)
raise
if len(action_sequence_fields
) >= self._max_offline_logical_forms:
break
if not action_sequence_fields:
# This is not great, but we're only doing it when we're passed logical form
# supervision, so we're expecting labeled logical forms, but we can't actually
# produce the logical forms. We should skip this instance. Note that this affects
# _dev_ and _test_ instances, too, so your metrics could be over-estimates on the
# full test data.
return None
fields['target_action_sequences'] = ListField(
action_sequence_fields)
if self._output_agendas:
agenda_index_fields: List[Field] = []
for agenda_string in world.get_agenda(conservative=True):
agenda_index_fields.append(
IndexField(action_map[agenda_string], action_field))
if not agenda_index_fields:
agenda_index_fields = [IndexField(-1, action_field)]
fields['agenda'] = ListField(agenda_index_fields)
return Instance(fields)