def forward( self, # type: ignore question: Dict[str, torch.LongTensor], table: Dict[str, torch.LongTensor], world: List[WikiTablesLanguage], actions: List[List[ProductionRule]], agenda: torch.LongTensor, target_values: List[List[str]] = None, metadata: List[Dict[str, Any]] = None, ) -> Dict[str, torch.Tensor]: """ Parameters ---------- question : Dict[str, torch.LongTensor] The output of ``TextField.as_array()`` applied on the question ``TextField``. This will be passed through a ``TextFieldEmbedder`` and then through an encoder. table : ``Dict[str, torch.LongTensor]`` The output of ``KnowledgeGraphField.as_array()`` applied on the table ``KnowledgeGraphField``. This output is similar to a ``TextField`` output, where each entity in the table is treated as a "token", and we will use a ``TextFieldEmbedder`` to get embeddings for each entity. world : ``List[WikiTablesLanguage]`` We use a ``MetadataField`` to get the ``WikiTablesLanguage`` object for each input instance. Because of how ``MetadataField`` works, this gets passed to us as a ``List[WikiTablesLanguage]``, actions : ``List[List[ProductionRule]]`` A list of all possible actions for each ``world`` in the batch, indexed into a ``ProductionRule`` using a ``ProductionRuleField``. We will embed all of these and use the embeddings to determine which action to take at each timestep in the decoder. agenda : ``torch.LongTensor`` Agenda vectors that the checklist vectors will be compared against to compute the checklist cost. target_values : ``List[List[str]]``, optional (default = None) For each instance, a list of target values taken from the example lisp string. We pass this list to the evaluator along with logical forms to compute denotation accuracy. metadata : ``List[Dict[str, Any]]``, optional (default = None) Metadata containing the original tokenized question within a 'question_tokens' field. """ # Each instance's agenda is of size (agenda_size, 1) agenda_list = [a for a in agenda] checklist_states = [] all_terminal_productions = [ set(instance_world.terminal_productions.values()) for instance_world in world ] max_num_terminals = max( [len(terminals) for terminals in all_terminal_productions]) for instance_actions, instance_agenda, terminal_productions in zip( actions, agenda_list, all_terminal_productions): checklist_info = self._get_checklist_info(instance_agenda, instance_actions, terminal_productions, max_num_terminals) checklist_target, terminal_actions, checklist_mask = checklist_info initial_checklist = checklist_target.new_zeros( checklist_target.size()) checklist_states.append( ChecklistStatelet( terminal_actions=terminal_actions, checklist_target=checklist_target, checklist_mask=checklist_mask, checklist=initial_checklist, )) outputs: Dict[str, Any] = {} rnn_state, grammar_state = self._get_initial_rnn_and_grammar_state( question, table, world, actions, outputs) batch_size = len(rnn_state) initial_score = rnn_state[0].hidden_state.new_zeros(batch_size) initial_score_list = [initial_score[i] for i in range(batch_size)] initial_state = CoverageState( batch_indices=list(range(batch_size)), # type: ignore action_history=[[] for _ in range(batch_size)], score=initial_score_list, rnn_state=rnn_state, grammar_state=grammar_state, checklist_state=checklist_states, possible_actions=actions, extras=target_values, debug_info=None, ) if target_values is not None: logger.warning(f"TARGET VALUES: {target_values}") trainer_outputs = self._decoder_trainer.decode( # type: ignore initial_state, self._decoder_step, partial(self._get_state_cost, world)) outputs.update(trainer_outputs) else: initial_state.debug_info = [[] for _ in range(batch_size)] batch_size = len(actions) agenda_indices = [actions_[:, 0].cpu().data for actions_ in agenda] action_mapping = {} for batch_index, batch_actions in enumerate(actions): for action_index, action in enumerate(batch_actions): action_mapping[(batch_index, action_index)] = action[0] best_final_states = self._beam_search.search( self._max_decoding_steps, initial_state, self._decoder_step, keep_final_unfinished_states=False, ) for i in range(batch_size): in_agenda_ratio = 0.0 # Decoding may not have terminated with any completed logical forms, if `num_steps` # isn't long enough (or if the model is not trained enough and gets into an # infinite action loop). if i in best_final_states: action_sequence = best_final_states[i][0].action_history[0] action_strings = [ action_mapping[(i, action_index)] for action_index in action_sequence ] instance_possible_actions = actions[i] agenda_actions = [] for rule_id in agenda_indices[i]: rule_id = int(rule_id) if rule_id == -1: continue action_string = instance_possible_actions[rule_id][0] agenda_actions.append(action_string) actions_in_agenda = [ action in action_strings for action in agenda_actions ] if actions_in_agenda: # Note: This means that when there are no actions on agenda, agenda coverage # will be 0, not 1. in_agenda_ratio = sum(actions_in_agenda) / len( actions_in_agenda) self._agenda_coverage(in_agenda_ratio) self._compute_validation_outputs(actions, best_final_states, world, target_values, metadata, outputs) return outputs
def forward( self, # type: ignore sentence: Dict[str, torch.LongTensor], worlds: List[List[NlvrLanguage]], actions: List[List[ProductionRule]], agenda: torch.LongTensor, identifier: List[str] = None, labels: torch.LongTensor = None, epoch_num: List[int] = None, metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ Decoder logic for producing type constrained target sequences that maximize coverage of their respective agendas, and minimize a denotation based loss. """ # We look at the epoch number and adjust the checklist cost weight if needed here. instance_epoch_num = epoch_num[0] if epoch_num is not None else None if self._dynamic_cost_rate is not None: if self.training and instance_epoch_num is None: raise RuntimeError( "If you want a dynamic cost weight, use the " "BucketIterator with track_epoch=True.") if instance_epoch_num != self._last_epoch_in_forward: if instance_epoch_num >= self._dynamic_cost_wait_epochs: decrement = self._checklist_cost_weight * self._dynamic_cost_rate self._checklist_cost_weight -= decrement logger.info("Checklist cost weight is now %f", self._checklist_cost_weight) self._last_epoch_in_forward = instance_epoch_num batch_size = len(worlds) initial_rnn_state = self._get_initial_rnn_state(sentence) initial_score_list = [ next(iter(sentence.values())).new_zeros(1, dtype=torch.float) for i in range(batch_size) ] # TODO (pradeep): Assuming all worlds give the same set of valid actions. initial_grammar_state = [ self._create_grammar_state(worlds[i][0], actions[i]) for i in range(batch_size) ] label_strings = self._get_label_strings( labels) if labels is not None else None # Each instance's agenda is of size (agenda_size, 1) # TODO(mattg): It looks like the agenda is only ever used on the CPU. In that case, it's a # waste to copy it to the GPU and then back, and this should probably be a MetadataField. agenda_list = [agenda[i] for i in range(batch_size)] initial_checklist_states = [] for instance_actions, instance_agenda in zip(actions, agenda_list): checklist_info = self._get_checklist_info(instance_agenda, instance_actions) checklist_target, terminal_actions, checklist_mask = checklist_info initial_checklist = checklist_target.new_zeros( checklist_target.size()) initial_checklist_states.append( ChecklistStatelet(terminal_actions=terminal_actions, checklist_target=checklist_target, checklist_mask=checklist_mask, checklist=initial_checklist)) initial_state = CoverageState( batch_indices=list(range(batch_size)), action_history=[[] for _ in range(batch_size)], score=initial_score_list, rnn_state=initial_rnn_state, grammar_state=initial_grammar_state, possible_actions=actions, extras=label_strings, checklist_state=initial_checklist_states) if not self.training: initial_state.debug_info = [[] for _ in range(batch_size)] agenda_data = [agenda_[:, 0].cpu().data for agenda_ in agenda_list] outputs = self._decoder_trainer.decode( initial_state, # type: ignore self._decoder_step, partial(self._get_state_cost, worlds)) if identifier is not None: outputs['identifier'] = identifier best_final_states = outputs['best_final_states'] best_action_sequences = {} for batch_index, states in best_final_states.items(): best_action_sequences[batch_index] = [ state.action_history[0] for state in states ] batch_action_strings = self._get_action_strings( actions, best_action_sequences) batch_denotations = self._get_denotations(batch_action_strings, worlds) if labels is not None: # We're either training or validating. self._update_metrics(action_strings=batch_action_strings, worlds=worlds, label_strings=label_strings, possible_actions=actions, agenda_data=agenda_data) else: # We're testing. if metadata is not None: outputs["sentence_tokens"] = [ x["sentence_tokens"] for x in metadata ] outputs['debug_info'] = [] for i in range(batch_size): outputs['debug_info'].append( best_final_states[i][0].debug_info[0]) # type: ignore outputs["best_action_strings"] = batch_action_strings outputs["denotations"] = batch_denotations action_mapping = {} for batch_index, batch_actions in enumerate(actions): for action_index, action in enumerate(batch_actions): action_mapping[(batch_index, action_index)] = action[0] outputs['action_mapping'] = action_mapping return outputs
def forward( self, # type: ignore sentence: Dict[str, torch.LongTensor], worlds: List[List[NlvrLanguage]], actions: List[List[ProductionRule]], agenda: torch.LongTensor, identifier: List[str] = None, labels: torch.LongTensor = None, metadata: List[Dict[str, Any]] = None, ) -> Dict[str, torch.Tensor]: """ Decoder logic for producing type constrained target sequences that maximize coverage of their respective agendas, and minimize a denotation based loss. """ if self._dynamic_cost_rate is not None: # This could be added back pretty easily with an EpochCallback passed to the Trainer (it # just has to set the epoch number on the model, which could then be queried in here). logger.warning( "Dynamic cost rate functionality was removed in AllenNLP 1.0. If you want this, " "use version 0.9. We will just use the static checklist cost weight." ) batch_size = len(worlds) initial_rnn_state = self._get_initial_rnn_state(sentence) initial_score_list = [agenda.new_zeros(1, dtype=torch.float) for i in range(batch_size)] # TODO (pradeep): Assuming all worlds give the same set of valid actions. initial_grammar_state = [ self._create_grammar_state(worlds[i][0], actions[i]) for i in range(batch_size) ] label_strings = self._get_label_strings(labels) if labels is not None else None # Each instance's agenda is of size (agenda_size, 1) # TODO(mattg): It looks like the agenda is only ever used on the CPU. In that case, it's a # waste to copy it to the GPU and then back, and this should probably be a MetadataField. agenda_list = [agenda[i] for i in range(batch_size)] initial_checklist_states = [] for instance_actions, instance_agenda in zip(actions, agenda_list): checklist_info = self._get_checklist_info(instance_agenda, instance_actions) checklist_target, terminal_actions, checklist_mask = checklist_info initial_checklist = checklist_target.new_zeros(checklist_target.size()) initial_checklist_states.append( ChecklistStatelet( terminal_actions=terminal_actions, checklist_target=checklist_target, checklist_mask=checklist_mask, checklist=initial_checklist, ) ) initial_state = CoverageState( batch_indices=list(range(batch_size)), action_history=[[] for _ in range(batch_size)], score=initial_score_list, rnn_state=initial_rnn_state, grammar_state=initial_grammar_state, possible_actions=actions, extras=label_strings, checklist_state=initial_checklist_states, ) if not self.training: initial_state.debug_info = [[] for _ in range(batch_size)] agenda_data = [agenda_[:, 0].cpu().data for agenda_ in agenda_list] outputs = self._decoder_trainer.decode( # type: ignore initial_state, self._decoder_step, partial(self._get_state_cost, worlds) ) if identifier is not None: outputs["identifier"] = identifier best_final_states = outputs["best_final_states"] best_action_sequences = {} for batch_index, states in best_final_states.items(): best_action_sequences[batch_index] = [state.action_history[0] for state in states] batch_action_strings = self._get_action_strings(actions, best_action_sequences) batch_denotations = self._get_denotations(batch_action_strings, worlds) if labels is not None: # We're either training or validating. self._update_metrics( action_strings=batch_action_strings, worlds=worlds, label_strings=label_strings, possible_actions=actions, agenda_data=agenda_data, ) else: # We're testing. if metadata is not None: outputs["sentence_tokens"] = [x["sentence_tokens"] for x in metadata] outputs["debug_info"] = [] for i in range(batch_size): outputs["debug_info"].append(best_final_states[i][0].debug_info[0]) # type: ignore outputs["best_action_strings"] = batch_action_strings outputs["denotations"] = batch_denotations action_mapping = {} for batch_index, batch_actions in enumerate(actions): for action_index, action in enumerate(batch_actions): action_mapping[(batch_index, action_index)] = action[0] outputs["action_mapping"] = action_mapping return outputs