def test_action_sequence_to_sql(self): action_sequence = [ 'statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> ["DISTINCT"]', "select_results -> [col_refs]", 'col_refs -> [col_ref, ",", col_refs]', 'col_ref -> ["city", ".", "city_code"]', "col_refs -> [col_ref]", 'col_ref -> ["city", ".", "city_name"]', "table_refs -> [table_name]", 'table_name -> ["city"]', 'where_clause -> ["WHERE", "(", conditions, ")"]', "conditions -> [condition]", "condition -> [biexpr]", 'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]', 'binaryop -> ["="]', "city_city_name_string -> [\"'BOSTON'\"]", ] sql_query = action_sequence_to_sql(action_sequence) assert (sql_query == "( SELECT DISTINCT city . city_code , city . city_name " "FROM city WHERE ( city . city_name = 'BOSTON' ) ) ;")
def test_action_sequence_to_sql(self): action_sequence = ['statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> ["DISTINCT"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref, ",", col_refs]', 'col_ref -> ["city", ".", "city_code"]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_name"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'where_clause -> ["WHERE", "(", conditions, ")"]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]', 'binaryop -> ["="]', 'city_city_name_string -> ["\'BOSTON\'"]'] sql_query = action_sequence_to_sql(action_sequence) assert sql_query == "( SELECT DISTINCT city . city_code , city . city_name " \ "FROM city WHERE ( city . city_name = 'BOSTON' ) ) ;"
def forward( self, # type: ignore tokens: Dict[str, torch.LongTensor], valid_actions: List[List[ProductionRule]], action_sequence: torch.LongTensor = None, spans: torch.IntTensor = None) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ We set up the initial state for the decoder, and pass that state off to either a DecoderTrainer, if we're training, or a BeamSearch for inference, if we're not. Parameters ---------- tokens : Dict[str, torch.LongTensor] The output of ``TextField.as_array()`` applied on the tokens ``TextField``. This will be passed through a ``TextFieldEmbedder`` and then through an encoder. valid_actions : ``List[List[ProductionRule]]`` A list of all possible actions for each ``World`` in the batch, indexed into a ``ProductionRule`` using a ``ProductionRuleField``. We will embed all of these and use the embeddings to determine which action to take at each timestep in the decoder. action_sequence : torch.Tensor, optional (default=None) The action sequence for the correct action sequence, where each action is an index into the list of possible actions. This tensor has shape ``(batch_size, sequence_length, 1)``. We remove the trailing dimension. spans: torch.Tensor, optional (default=None) A tensor of shape (batch_size, num_spans, 2), representing the inclusive start and end indices of input spans that could be informative for the decoder. Comes from a ``ListField[SpanField]`` """ encode_outputs = self._encode(tokens, spans) # encode_outputs['mask'] shape: (batch_size, num_tokens, encoder_output_dim) batch_size = encode_outputs['mask'].size(0) initial_state = self._get_initial_state( encode_outputs['encoder_outputs'], encode_outputs['mask'], valid_actions) if action_sequence is not None: # Remove the trailing dimension (from ListField[ListField[IndexField]]). action_sequence = action_sequence.squeeze(-1) target_mask = action_sequence != self._action_padding_index else: target_mask = None outputs: Dict[str, Any] = {} if action_sequence is not None: # target_action_sequence is of shape (batch_size, 1, target_sequence_length) # here after we unsqueeze it for the MML trainer. try: loss_output = self._decoder_trainer.decode( initial_state, self._transition_function, (action_sequence.unsqueeze(1), target_mask.unsqueeze(1))) except ZeroDivisionError as e: logger.info( f"Input utterance in ZeroDivisionError: {[t.text for t in tokens['tokens']]}" ) raise e outputs.update(loss_output) if not self.training: action_mapping = [] for batch_actions in valid_actions: batch_action_mapping = {} for action_index, action in enumerate(batch_actions): batch_action_mapping[action_index] = action[0] action_mapping.append(batch_action_mapping) outputs['action_mapping'] = action_mapping # This tells the state to start keeping track of debug info, which we'll pass along in # our output dictionary. initial_state.debug_info = [[] for _ in range(batch_size)] best_final_states = self._beam_search.search( self._max_decoding_steps, initial_state, self._transition_function, keep_final_unfinished_states=True) outputs['best_action_sequence'] = [] outputs['debug_info'] = [] outputs['predicted_sql_query'] = [] outputs['target_sql_query'] = [] outputs['sql_queries'] = [] for i in range(batch_size): # Add the target sql from the target actions for sql tokens exact match comparison target_sql_query = '' if action_sequence is not None: target_action_strings = [ action_mapping[i][action_index] for action_index in action_sequence[i].data.tolist() if action_index != self._action_padding_index ] target_sql_query = action_sequence_to_sql( target_action_strings) # target_sql_query = sqlparse.format(target_sql_query, reindent=True) target_sql_query_for_acc = target_sql_query.split() # Decoding may not have terminated with any completed valid SQL queries, if `num_steps` # isn't long enough (or if the model is not trained enough and gets into an # infinite action loop). if i not in best_final_states: self._exact_match(0) self._action_similarity(0) outputs['target_sql_query'].append( target_sql_query_for_acc) outputs['predicted_sql_query'].append('') continue best_action_indices = best_final_states[i][0].action_history[0] action_strings = [ action_mapping[i][action_index] for action_index in best_action_indices ] predicted_sql_query = action_sequence_to_sql(action_strings) predicted_sql_query_for_acc = predicted_sql_query.split() if action_sequence is not None: # Use a Tensor, not a Variable, to avoid a memory leak. targets = action_sequence[i].data sequence_in_targets = 0 sequence_in_targets = self._action_history_match( best_action_indices, targets) self._exact_match(sequence_in_targets) similarity = difflib.SequenceMatcher( None, best_action_indices, targets) self._action_similarity(similarity.ratio()) # predicted_sql_query_for_acc = [token if '@' not in token else token.split('@')[1] for token in # predicted_sql_query.split()] # target_sql_query_for_acc = [token if '@' not in token else token.split('@')[1] for token in # target_sql_query.split()] predicted_sql_query_for_acc = re.sub( r" TABLE_PLACEHOLDER AS ([A-Z_]+)\s*(alias[0-9]) ", r" \g<1> AS \g<1>\g<2> ", predicted_sql_query).split() target_sql_query_for_acc = re.sub( r" TABLE_PLACEHOLDER AS ([A-Z_]+)\s*(alias[0-9]) ", r" \g<1> AS \g<1>\g<2> ", target_sql_query).split() self._valid_sql_query([predicted_sql_query_for_acc], [target_sql_query_for_acc]) self._token_match([predicted_sql_query_for_acc], [target_sql_query_for_acc]) self._kb_match([predicted_sql_query_for_acc], [target_sql_query_for_acc]) self._schema_free_match([predicted_sql_query_for_acc], [target_sql_query_for_acc]) outputs['best_action_sequence'].append(action_strings) # outputs['predicted_sql_query'].append(sqlparse.format(predicted_sql_query, reindent=True)) outputs['predicted_sql_query'].append( predicted_sql_query_for_acc) outputs['target_sql_query'].append(target_sql_query_for_acc) outputs['debug_info'].append( best_final_states[i][0].debug_info[0]) # type: ignore return outputs
def forward( self, # type: ignore tokens: Dict[str, torch.LongTensor], valid_actions: List[List[ProductionRule]], action_sequence: torch.LongTensor = None ) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ We set up the initial state for the decoder, and pass that state off to either a DecoderTrainer, if we're training, or a BeamSearch for inference, if we're not. Parameters ---------- tokens : Dict[str, torch.LongTensor] The output of ``TextField.as_array()`` applied on the tokens ``TextField``. This will be passed through a ``TextFieldEmbedder`` and then through an encoder. valid_actions : ``List[List[ProductionRule]]`` A list of all possible actions for each ``World`` in the batch, indexed into a ``ProductionRule`` using a ``ProductionRuleField``. We will embed all of these and use the embeddings to determine which action to take at each timestep in the decoder. action_sequence : torch.Tensor, optional (default=None) The action sequence for the correct action sequence, where each action is an index into the list of possible actions. This tensor has shape ``(batch_size, sequence_length, 1)``. We remove the trailing dimension. """ embedded_utterance = self._utterance_embedder(tokens) mask = util.get_text_field_mask(tokens).float() batch_size = embedded_utterance.size(0) # (batch_size, num_tokens, encoder_output_dim) encoder_outputs = self._dropout(self._encoder(embedded_utterance, mask)) initial_state = self._get_initial_state(encoder_outputs, mask, valid_actions) if action_sequence is not None: # Remove the trailing dimension (from ListField[ListField[IndexField]]). action_sequence = action_sequence.squeeze(-1) target_mask = action_sequence != self._action_padding_index else: target_mask = None outputs: Dict[str, Any] = {} if action_sequence is not None: # target_action_sequence is of shape (batch_size, 1, target_sequence_length) # here after we unsqueeze it for the MML trainer. loss_output = self._decoder_trainer.decode( initial_state, self._transition_function, (action_sequence.unsqueeze(1), target_mask.unsqueeze(1))) outputs.update(loss_output) if not self.training: action_mapping = [] for batch_actions in valid_actions: batch_action_mapping = {} for action_index, action in enumerate(batch_actions): batch_action_mapping[action_index] = action[0] action_mapping.append(batch_action_mapping) outputs['action_mapping'] = action_mapping # This tells the state to start keeping track of debug info, which we'll pass along in # our output dictionary. initial_state.debug_info = [[] for _ in range(batch_size)] best_final_states = self._beam_search.search( self._max_decoding_steps, initial_state, self._transition_function, keep_final_unfinished_states=True) outputs['best_action_sequence'] = [] outputs['debug_info'] = [] outputs['predicted_sql_query'] = [] outputs['sql_queries'] = [] for i in range(batch_size): # Decoding may not have terminated with any completed valid SQL queries, if `num_steps` # isn't long enough (or if the model is not trained enough and gets into an # infinite action loop). if i not in best_final_states: self._exact_match(0) self._denotation_accuracy(0) self._valid_sql_query(0) self._action_similarity(0) outputs['predicted_sql_query'].append('') continue best_action_indices = best_final_states[i][0].action_history[0] action_strings = [ action_mapping[i][action_index] for action_index in best_action_indices ] predicted_sql_query = action_sequence_to_sql(action_strings) if action_sequence is not None: # Use a Tensor, not a Variable, to avoid a memory leak. targets = action_sequence[i].data sequence_in_targets = 0 sequence_in_targets = self._action_history_match( best_action_indices, targets) self._exact_match(sequence_in_targets) similarity = difflib.SequenceMatcher( None, best_action_indices, targets) self._action_similarity(similarity.ratio()) outputs['best_action_sequence'].append(action_strings) outputs['predicted_sql_query'].append( sqlparse.format(predicted_sql_query, reindent=True)) outputs['debug_info'].append( best_final_states[i][0].debug_info[0]) # type: ignore return outputs
def forward( self, # type: ignore utterance: Dict[str, torch.LongTensor], world: List[AtisWorld], actions: List[List[ProductionRule]], linking_scores: torch.Tensor, target_action_sequence: torch.LongTensor = None, sql_queries: List[List[str]] = None, ) -> Dict[str, torch.Tensor]: """ We set up the initial state for the decoder, and pass that state off to either a DecoderTrainer, if we're training, or a BeamSearch for inference, if we're not. Parameters ---------- utterance : Dict[str, torch.LongTensor] The output of ``TextField.as_array()`` applied on the utterance ``TextField``. This will be passed through a ``TextFieldEmbedder`` and then through an encoder. world : ``List[AtisWorld]`` We use a ``MetadataField`` to get the ``World`` for each input instance. Because of how ``MetadataField`` works, this gets passed to us as a ``List[AtisWorld]``, actions : ``List[List[ProductionRule]]`` A list of all possible actions for each ``World`` in the batch, indexed into a ``ProductionRule`` using a ``ProductionRuleField``. We will embed all of these and use the embeddings to determine which action to take at each timestep in the decoder. linking_scores: ``torch.Tensor`` A matrix of the linking the utterance tokens and the entities. This is a binary matrix that is deterministically generated where each entry indicates whether a token generated an entity. This tensor has shape ``(batch_size, num_entities, num_utterance_tokens)``. target_action_sequence : torch.Tensor, optional (default=None) The action sequence for the correct action sequence, where each action is an index into the list of possible actions. This tensor has shape ``(batch_size, sequence_length, 1)``. We remove the trailing dimension. sql_queries : List[List[str]], optional (default=None) A list of the SQL queries that are given during training or validation. """ initial_state = self._get_initial_state(utterance, world, actions, linking_scores) batch_size = linking_scores.shape[0] if target_action_sequence is not None: # Remove the trailing dimension (from ListField[ListField[IndexField]]). target_action_sequence = target_action_sequence.squeeze(-1) target_mask = target_action_sequence != self._action_padding_index else: target_mask = None if self.training: # target_action_sequence is of shape (batch_size, 1, sequence_length) here after we unsqueeze it for # the MML trainer. return self._decoder_trainer.decode( initial_state, self._transition_function, (target_action_sequence.unsqueeze(1), target_mask.unsqueeze(1)), ) else: # TODO(kevin) Move some of this functionality to a separate method for computing validation outputs. action_mapping = {} for batch_index, batch_actions in enumerate(actions): for action_index, action in enumerate(batch_actions): action_mapping[(batch_index, action_index)] = action[0] outputs: Dict[str, Any] = {"action_mapping": action_mapping} outputs["linking_scores"] = linking_scores if target_action_sequence is not None: outputs["loss"] = self._decoder_trainer.decode( initial_state, self._transition_function, (target_action_sequence.unsqueeze(1), target_mask.unsqueeze(1)), )["loss"] num_steps = self._max_decoding_steps # This tells the state to start keeping track of debug info, which we'll pass along in # our output dictionary. initial_state.debug_info = [[] for _ in range(batch_size)] best_final_states = self._beam_search.search( num_steps, initial_state, self._transition_function, keep_final_unfinished_states=False, ) outputs["best_action_sequence"] = [] outputs["debug_info"] = [] outputs["entities"] = [] outputs["predicted_sql_query"] = [] outputs["sql_queries"] = [] outputs["utterance"] = [] outputs["tokenized_utterance"] = [] for i in range(batch_size): # Decoding may not have terminated with any completed valid SQL queries, if `num_steps` # isn't long enough (or if the model is not trained enough and gets into an # infinite action loop). if i not in best_final_states: self._exact_match(0) self._denotation_accuracy(0) self._valid_sql_query(0) self._action_similarity(0) outputs["predicted_sql_query"].append("") continue best_action_indices = best_final_states[i][0].action_history[0] action_strings = [ action_mapping[(i, action_index)] for action_index in best_action_indices ] predicted_sql_query = action_sequence_to_sql(action_strings) if target_action_sequence is not None: # Use a Tensor, not a Variable, to avoid a memory leak. targets = target_action_sequence[i].data sequence_in_targets = 0 sequence_in_targets = self._action_history_match( best_action_indices, targets) self._exact_match(sequence_in_targets) similarity = difflib.SequenceMatcher( None, best_action_indices, targets) self._action_similarity(similarity.ratio()) if sql_queries and sql_queries[i]: denotation_correct = self._executor.evaluate_sql_query( predicted_sql_query, sql_queries[i]) self._denotation_accuracy(denotation_correct) outputs["sql_queries"].append(sql_queries[i]) outputs["utterance"].append(world[i].utterances[-1]) outputs["tokenized_utterance"].append([ token.text for token in world[i].tokenized_utterances[-1] ]) outputs["entities"].append(world[i].entities) outputs["best_action_sequence"].append(action_strings) outputs["predicted_sql_query"].append( sqlparse.format(predicted_sql_query, reindent=True)) outputs["debug_info"].append( best_final_states[i][0].debug_info[0]) # type: ignore return outputs
def forward(self, # type: ignore tokens: Dict[str, torch.LongTensor], valid_actions: List[List[ProductionRule]], action_sequence: torch.LongTensor = None) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ We set up the initial state for the decoder, and pass that state off to either a DecoderTrainer, if we're training, or a BeamSearch for inference, if we're not. Parameters ---------- tokens : Dict[str, torch.LongTensor] The output of ``TextField.as_array()`` applied on the tokens ``TextField``. This will be passed through a ``TextFieldEmbedder`` and then through an encoder. valid_actions : ``List[List[ProductionRule]]`` A list of all possible actions for each ``World`` in the batch, indexed into a ``ProductionRule`` using a ``ProductionRuleField``. We will embed all of these and use the embeddings to determine which action to take at each timestep in the decoder. target_action_sequence : torch.Tensor, optional (default=None) The action sequence for the correct action sequence, where each action is an index into the list of possible actions. This tensor has shape ``(batch_size, sequence_length, 1)``. We remove the trailing dimension. sql_queries : List[List[str]], optional (default=None) A list of the SQL queries that are given during training or validation. """ embedded_utterance = self._utterance_embedder(tokens) mask = util.get_text_field_mask(tokens).float() batch_size = embedded_utterance.size(0) # (batch_size, num_tokens, encoder_output_dim) encoder_outputs = self._dropout(self._encoder(embedded_utterance, mask)) initial_state = self._get_initial_state(encoder_outputs, mask, valid_actions) if action_sequence is not None: # Remove the trailing dimension (from ListField[ListField[IndexField]]). action_sequence = action_sequence.squeeze(-1) target_mask = action_sequence != self._action_padding_index else: target_mask = None outputs: Dict[str, Any] = {} if action_sequence is not None: # target_action_sequence is of shape (batch_size, 1, target_sequence_length) # here after we unsqueeze it for the MML trainer. loss_output = self._decoder_trainer.decode(initial_state, self._transition_function, (action_sequence.unsqueeze(1), target_mask.unsqueeze(1))) outputs.update(loss_output) if not self.training: action_mapping = [] for batch_actions in valid_actions: batch_action_mapping = {} for action_index, action in enumerate(batch_actions): batch_action_mapping[action_index] = action[0] action_mapping.append(batch_action_mapping) outputs['action_mapping'] = action_mapping # This tells the state to start keeping track of debug info, which we'll pass along in # our output dictionary. initial_state.debug_info = [[] for _ in range(batch_size)] best_final_states = self._beam_search.search(self._max_decoding_steps, initial_state, self._transition_function, keep_final_unfinished_states=True) outputs['best_action_sequence'] = [] outputs['debug_info'] = [] outputs['predicted_sql_query'] = [] outputs['sql_queries'] = [] for i in range(batch_size): # Decoding may not have terminated with any completed valid SQL queries, if `num_steps` # isn't long enough (or if the model is not trained enough and gets into an # infinite action loop). if i not in best_final_states: self._exact_match(0) self._denotation_accuracy(0) self._valid_sql_query(0) self._action_similarity(0) outputs['predicted_sql_query'].append('') continue best_action_indices = best_final_states[i][0].action_history[0] action_strings = [action_mapping[i][action_index] for action_index in best_action_indices] predicted_sql_query = action_sequence_to_sql(action_strings) if action_sequence is not None: # Use a Tensor, not a Variable, to avoid a memory leak. targets = action_sequence[i].data sequence_in_targets = 0 sequence_in_targets = self._action_history_match(best_action_indices, targets) self._exact_match(sequence_in_targets) similarity = difflib.SequenceMatcher(None, best_action_indices, targets) self._action_similarity(similarity.ratio()) outputs['best_action_sequence'].append(action_strings) outputs['predicted_sql_query'].append(sqlparse.format(predicted_sql_query, reindent=True)) outputs['debug_info'].append(best_final_states[i][0].debug_info[0]) # type: ignore return outputs
def forward(self, # type: ignore utterance: Dict[str, torch.LongTensor], world: List[AtisWorld], actions: List[List[ProductionRule]], linking_scores: torch.Tensor, target_action_sequence: torch.LongTensor = None, sql_queries: List[List[str]] = None) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ We set up the initial state for the decoder, and pass that state off to either a DecoderTrainer, if we're training, or a BeamSearch for inference, if we're not. Parameters ---------- utterance : Dict[str, torch.LongTensor] The output of ``TextField.as_array()`` applied on the utterance ``TextField``. This will be passed through a ``TextFieldEmbedder`` and then through an encoder. world : ``List[AtisWorld]`` We use a ``MetadataField`` to get the ``World`` for each input instance. Because of how ``MetadataField`` works, this gets passed to us as a ``List[AtisWorld]``, actions : ``List[List[ProductionRule]]`` A list of all possible actions for each ``World`` in the batch, indexed into a ``ProductionRule`` using a ``ProductionRuleField``. We will embed all of these and use the embeddings to determine which action to take at each timestep in the decoder. linking_scores: ``torch.Tensor`` A matrix of the linking the utterance tokens and the entities. This is a binary matrix that is deterministically generated where each entry indicates whether a token generated an entity. This tensor has shape ``(batch_size, num_entities, num_utterance_tokens)``. target_action_sequence : torch.Tensor, optional (default=None) The action sequence for the correct action sequence, where each action is an index into the list of possible actions. This tensor has shape ``(batch_size, sequence_length, 1)``. We remove the trailing dimension. sql_queries : List[List[str]], optional (default=None) A list of the SQL queries that are given during training or validation. """ initial_state = self._get_initial_state(utterance, world, actions, linking_scores) batch_size = linking_scores.shape[0] if target_action_sequence is not None: # Remove the trailing dimension (from ListField[ListField[IndexField]]). target_action_sequence = target_action_sequence.squeeze(-1) target_mask = target_action_sequence != self._action_padding_index else: target_mask = None if self.training: # target_action_sequence is of shape (batch_size, 1, sequence_length) here after we unsqueeze it for # the MML trainer. return self._decoder_trainer.decode(initial_state, self._transition_function, (target_action_sequence.unsqueeze(1), target_mask.unsqueeze(1))) else: # TODO(kevin) Move some of this functionality to a separate method for computing validation outputs. action_mapping = {} for batch_index, batch_actions in enumerate(actions): for action_index, action in enumerate(batch_actions): action_mapping[(batch_index, action_index)] = action[0] outputs: Dict[str, Any] = {'action_mapping': action_mapping} outputs['linking_scores'] = linking_scores if target_action_sequence is not None: outputs['loss'] = self._decoder_trainer.decode(initial_state, self._transition_function, (target_action_sequence.unsqueeze(1), target_mask.unsqueeze(1)))['loss'] num_steps = self._max_decoding_steps # This tells the state to start keeping track of debug info, which we'll pass along in # our output dictionary. initial_state.debug_info = [[] for _ in range(batch_size)] best_final_states = self._beam_search.search(num_steps, initial_state, self._transition_function, keep_final_unfinished_states=False) outputs['best_action_sequence'] = [] outputs['debug_info'] = [] outputs['entities'] = [] outputs['predicted_sql_query'] = [] outputs['sql_queries'] = [] outputs['utterance'] = [] outputs['tokenized_utterance'] = [] for i in range(batch_size): # Decoding may not have terminated with any completed valid SQL queries, if `num_steps` # isn't long enough (or if the model is not trained enough and gets into an # infinite action loop). if i not in best_final_states: self._exact_match(0) self._denotation_accuracy(0) self._valid_sql_query(0) self._action_similarity(0) outputs['predicted_sql_query'].append('') continue best_action_indices = best_final_states[i][0].action_history[0] action_strings = [action_mapping[(i, action_index)] for action_index in best_action_indices] predicted_sql_query = action_sequence_to_sql(action_strings) if target_action_sequence is not None: # Use a Tensor, not a Variable, to avoid a memory leak. targets = target_action_sequence[i].data sequence_in_targets = 0 sequence_in_targets = self._action_history_match(best_action_indices, targets) self._exact_match(sequence_in_targets) similarity = difflib.SequenceMatcher(None, best_action_indices, targets) self._action_similarity(similarity.ratio()) if sql_queries and sql_queries[i]: denotation_correct = self._executor.evaluate_sql_query(predicted_sql_query, sql_queries[i]) self._denotation_accuracy(denotation_correct) outputs['sql_queries'].append(sql_queries[i]) outputs['utterance'].append(world[i].utterances[-1]) outputs['tokenized_utterance'].append([token.text for token in world[i].tokenized_utterances[-1]]) outputs['entities'].append(world[i].entities) outputs['best_action_sequence'].append(action_strings) outputs['predicted_sql_query'].append(sqlparse.format(predicted_sql_query, reindent=True)) outputs['debug_info'].append(best_final_states[i][0].debug_info[0]) # type: ignore return outputs