def test_weighted_sum_works_on_simple_input(self): batch_size = 1 sentence_length = 5 embedding_dim = 4 sentence_array = numpy.random.rand(batch_size, sentence_length, embedding_dim) sentence_tensor = torch.from_numpy(sentence_array).float() attention_tensor = torch.FloatTensor([[.3, .4, .1, 0, 1.2]]) aggregated_array = util.weighted_sum(sentence_tensor, attention_tensor).data.numpy() assert aggregated_array.shape == (batch_size, embedding_dim) expected_array = (0.3 * sentence_array[0, 0] + 0.4 * sentence_array[0, 1] + 0.1 * sentence_array[0, 2] + 0.0 * sentence_array[0, 3] + 1.2 * sentence_array[0, 4]) numpy.testing.assert_almost_equal(aggregated_array, [expected_array], decimal=5)
def test_weighted_sum_handles_higher_order_input(self): batch_size = 1 length_1 = 5 length_2 = 6 length_3 = 2 embedding_dim = 4 sentence_array = numpy.random.rand(batch_size, length_1, length_2, length_3, embedding_dim) attention_array = numpy.random.rand(batch_size, length_1, length_2, length_3) sentence_tensor = torch.from_numpy(sentence_array).float() attention_tensor = torch.from_numpy(attention_array).float() aggregated_array = util.weighted_sum(sentence_tensor, attention_tensor).data.numpy() assert aggregated_array.shape == (batch_size, length_1, length_2, embedding_dim) expected_array = (attention_array[0, 3, 2, 0] * sentence_array[0, 3, 2, 0] + attention_array[0, 3, 2, 1] * sentence_array[0, 3, 2, 1]) numpy.testing.assert_almost_equal(aggregated_array[0, 3, 2], expected_array, decimal=5)
def test_weighted_sum_handles_3d_attention_with_3d_matrix(self): batch_size = 1 length_1 = 5 length_2 = 2 embedding_dim = 4 sentence_array = numpy.random.rand(batch_size, length_2, embedding_dim) attention_array = numpy.random.rand(batch_size, length_1, length_2) sentence_tensor = torch.from_numpy(sentence_array).float() attention_tensor = torch.from_numpy(attention_array).float() aggregated_array = util.weighted_sum(sentence_tensor, attention_tensor).data.numpy() assert aggregated_array.shape == (batch_size, length_1, embedding_dim) for i in range(length_1): expected_array = (attention_array[0, i, 0] * sentence_array[0, 0] + attention_array[0, i, 1] * sentence_array[0, 1]) numpy.testing.assert_almost_equal(aggregated_array[0, i], expected_array, decimal=5)
def forward(self, X, mask=None, verbose=False): ''' Generate predictions Parameters ---------- X: input with shape (batch_size, max_seq_len, input_dim) mask: input with shape (batch_size, max_seq_len) ''' # Batch size batch_size = X.shape[0] # Batch vector (repeat across first dimension) vector = self.vector.unsqueeze(0).repeat(batch_size, 1) # if self.use_ffnn: Q = self.ffnn(X) else: Q = X # Attention weights # shape: (batch_size, max_seq_len) alphas = self.attention( \ vector = vector, matrix = Q, matrix_mask = mask) # Attended input # shape: (batch_size, encoder_query_dim) output = weighted_sum(X, alphas) # Dropout layer output = self.drop_layer(output) if verbose: logging.info('Attention') logging.info('\tinput_dim: {}'.format(input_dim)) logging.info('\tquery_dim: {}'.format(query_dim)) logging.info('\tactivation: {}'.format(activation)) logging.info('\tdropout: {}'.format(dropout)) logging.info('\tuse_ffnn: {}'.format(use_ffnn)) return (output, alphas)
def forward(self, tokens: torch.Tensor, mask: torch.Tensor): # pylint: disable=arguments-differ batch_size, sequence_length, _ = tokens.size() # Shape: (batch_size, sequence_length, sequence_length) similarity_matrix = self._matrix_attention(tokens, tokens) if self._num_attention_heads > 1: # In this case, the similarity matrix actually has shape # (batch_size, sequence_length, sequence_length, num_heads). To make the rest of the # logic below easier, we'll permute this to # (batch_size, sequence_length, num_heads, sequence_length). similarity_matrix = similarity_matrix.permute(0, 1, 3, 2) # Shape: (batch_size, sequence_length, [num_heads,] sequence_length) intra_sentence_attention = util.masked_softmax( similarity_matrix.contiguous(), mask) # Shape: (batch_size, sequence_length, projection_dim) output_token_representation = self._projection(tokens) if self._num_attention_heads > 1: # We need to split and permute the output representation to be # (batch_size, num_heads, sequence_length, projection_dim / num_heads), so that we can # do a proper weighted sum with `intra_sentence_attention`. shape = list(output_token_representation.size()) new_shape = shape[:-1] + [self._num_attention_heads, -1] # Shape: (batch_size, sequence_length, num_heads, projection_dim / num_heads) output_token_representation = output_token_representation.view( *new_shape) # Shape: (batch_size, num_heads, sequence_length, projection_dim / num_heads) output_token_representation = output_token_representation.permute( 0, 2, 1, 3) # Shape: (batch_size, sequence_length, [num_heads,] projection_dim [/ num_heads]) attended_sentence = util.weighted_sum(output_token_representation, intra_sentence_attention) if self._num_attention_heads > 1: # Here we concatenate the weighted representation for each head. We'll accomplish this # just with a resize. # Shape: (batch_size, sequence_length, projection_dim) attended_sentence = attended_sentence.view(batch_size, sequence_length, -1) # Shape: (batch_size, sequence_length, combination_dim) combined_tensors = util.combine_tensors(self._combination, [tokens, attended_sentence]) return self._output_projection(combined_tensors)
def _compute_attention( self, decoder_hidden_state: torch.LongTensor = None, encoder_outputs: torch.LongTensor = None, encoder_outputs_mask: torch.LongTensor = None) -> torch.Tensor: """Apply attention over encoder outputs and decoder state. Parameters ---------- decoder_hidden_state : ``torch.LongTensor`` A tensor of shape ``(batch_size, decoder_output_dim)``, which contains the current decoder hidden state to be used as the 'query' to the attention computation during the last time step. encoder_outputs : ``torch.LongTensor`` A tensor of shape ``(batch_size, max_input_sequence_length, encoder_output_dim)``, which contains all the encoder hidden states of the source tokens, i.e., the 'keys' to the attention computation encoder_mask : ``torch.LongTensor`` A tensor of shape (batch_size, max_input_sequence_length), which contains the mask of the encoded input. We want to avoid computing an attention score for positions of the source with zero-values (remember not all input sentences have the same length) Returns ------- torch.Tensor A tensor of shape (batch_size, encoder_output_dim) that contains the attended encoder outputs (aka context vector), i.e., we have ``applied`` the attention scores on the encoder hidden states. Notes ----- Don't forget to apply the final softmax over the **masked** encoder outputs! """ # Ensure mask is also a FloatTensor. Or else the multiplication within # attention will complain. # shape: (batch_size, max_input_sequence_length) encoder_outputs_mask = encoder_outputs_mask.float() # Main body of attention weights computation here attention_scores = encoder_outputs.bmm( decoder_hidden_state.unsqueeze(-1)).squeeze(-1) masked_attention_scores = masked_softmax(attention_scores, encoder_outputs_mask) attended_output = util.weighted_sum(encoder_outputs, masked_attention_scores) # masked_softmax() return attended_output, masked_attention_scores
def _prepare_decode_step_input( self, input_indices: torch.LongTensor, decoder_hidden_state: torch.LongTensor = None, encoder_outputs: torch.LongTensor = None, encoder_outputs_mask: torch.LongTensor = None) -> torch.LongTensor: """ Given the input indices for the current timestep of the decoder, and all the encoder outputs, compute the input at the current timestep. Note: This method is agnostic to whether the indices are gold indices or the predictions made by the decoder at the last timestep. So, this can be used even if we're doing some kind of scheduled sampling. If we're not using attention, the output of this method is just an embedding of the input indices. If we are, the output will be a concatentation of the embedding and an attended average of the encoder inputs. Parameters ---------- input_indices : torch.LongTensor Indices of either the gold inputs to the decoder or the predicted labels from the previous timestep. decoder_hidden_state : torch.LongTensor, optional (not needed if no attention) Output of from the decoder at the last time step. Needed only if using attention. encoder_outputs : torch.LongTensor, optional (not needed if no attention) Encoder outputs from all time steps. Needed only if using attention. encoder_outputs_mask : torch.LongTensor, optional (not needed if no attention) Masks on encoder outputs. Needed only if using attention. """ # input_indices : (batch_size,) since we are processing these one timestep at a time. # (batch_size, target_embedding_dim) embedded_input = self._target_embedder(input_indices) if self._attention_function: # encoder_outputs : (batch_size, input_sequence_length, encoder_output_dim) # Ensuring mask is also a FloatTensor. Or else the multiplication within attention will # complain. encoder_outputs_mask = encoder_outputs_mask.float() # (batch_size, input_sequence_length) input_weights = self._decoder_attention(decoder_hidden_state, encoder_outputs, encoder_outputs_mask) # (batch_size, encoder_output_dim) attended_input = weighted_sum(encoder_outputs, input_weights) # (batch_size, encoder_output_dim + target_embedding_dim) return torch.cat((attended_input, embedded_input), -1) else: return embedded_input
def _compute_attention(self, encoder_outputs: torch.Tensor, encoder_mask: torch.Tensor, decoder_outputs: torch.Tensor) -> torch.Tensor: """ Computes the attention-based decoder hidden representation by first computing the attention scores between the encoder and decoder hidden states, computing the attention context via a weighted average over the encoder hidden states, concatenating the decoder state with the context, and passing the result through the attention layer to project it back down to the decoder hidden state size. Parameters ---------- encoder_outputs: ``torch.Tensor``, ``(batch_size, num_document_tokens, encoder_hidden_size)`` The output from the encoder. encoder_mask: ``torch.Tensor``, ``(batch_size, num_document_tokens)`` The document token mask. decoder_outputs: ``torch.Tensor``, ``(batch_size, num_summary_tokens, decoder_hidden_size)`` The output from the decoder. Returns ------- hidden: ``torch.Tensor``, ``(batch_size, num_summary_tokens, decoder_hidden_size)`` The new decoder hidden state representation. attention_probabilities: ``torch.Tensor``, ``(batch_size, num_summary_tokens, num_document_tokens)`` The attention probabilities over the document tokens for each summary token """ # Compute the attention context # shape: (group_size, num_summary_tokens, num_document_tokens) attention_scores = self.attention(decoder_outputs, encoder_outputs) # shape: (group_size, num_summary_tokens, num_document_tokens) attention_probabilities = masked_softmax(attention_scores, encoder_mask) # shape: (group_size, num_summary_tokens, encoder_hidden_size) attention_context = weighted_sum(encoder_outputs, attention_probabilities) # Concatenate the attention context with the decoder outputs # then project back to the decoder hidden size # shape: (group_size, num_summary_tokens, encoder_hidden_size + decoder_hidden_size) concat = torch.cat([attention_context, decoder_outputs], dim=2) # shape: (group_size, num_summary_tokens, decoder_hidden_size) projected_hidden = self.attention_layer(concat) return projected_hidden, attention_probabilities
def _prepare_attended_input(self, decoder_hidden_state: torch.LongTensor = None, encoder_outputs: torch.LongTensor = None, encoder_outputs_mask: torch.LongTensor = None) -> torch.Tensor: """Apply attention over encoder outputs and decoder state.""" # Ensure mask is also a FloatTensor. Or else the multiplication within # attention will complain. # shape: (batch_size, max_input_sequence_length, encoder_output_dim) encoder_outputs_mask = encoder_outputs_mask.float() # shape: (batch_size, max_input_sequence_length) input_weights = self._attention( decoder_hidden_state, encoder_outputs, encoder_outputs_mask) # shape: (batch_size, encoder_output_dim) attended_input = util.weighted_sum(encoder_outputs, input_weights) return attended_input
def test_weighted_sum_handles_3d_attention_with_3d_matrix(self): batch_size = 1 length_1 = 5 length_2 = 2 embedding_dim = 4 sentence_array = numpy.random.rand(batch_size, length_2, embedding_dim) attention_array = numpy.random.rand(batch_size, length_1, length_2) sentence_tensor = Variable(torch.from_numpy(sentence_array).float()) attention_tensor = Variable(torch.from_numpy(attention_array).float()) aggregated_array = util.weighted_sum(sentence_tensor, attention_tensor).data.numpy() assert aggregated_array.shape == (batch_size, length_1, embedding_dim) for i in range(length_1): expected_array = (attention_array[0, i, 0] * sentence_array[0, 0] + attention_array[0, i, 1] * sentence_array[0, 1]) numpy.testing.assert_almost_equal(aggregated_array[0, i], expected_array, decimal=5)
def test_weighted_sum_works_on_simple_input(self): batch_size = 1 sentence_length = 5 embedding_dim = 4 sentence_array = numpy.random.rand(batch_size, sentence_length, embedding_dim) sentence_tensor = Variable(torch.from_numpy(sentence_array).float()) attention_tensor = Variable(torch.FloatTensor([[.3, .4, .1, 0, 1.2]])) aggregated_array = util.weighted_sum(sentence_tensor, attention_tensor).data.numpy() assert aggregated_array.shape == (batch_size, embedding_dim) expected_array = (0.3 * sentence_array[0, 0] + 0.4 * sentence_array[0, 1] + 0.1 * sentence_array[0, 2] + 0.0 * sentence_array[0, 3] + 1.2 * sentence_array[0, 4]) numpy.testing.assert_almost_equal(aggregated_array, [expected_array], decimal=5)
def test_weighted_sum_handles_uneven_higher_order_input(self): batch_size = 1 length_1 = 5 length_2 = 6 length_3 = 2 embedding_dim = 4 sentence_array = numpy.random.rand(batch_size, length_3, embedding_dim) attention_array = numpy.random.rand(batch_size, length_1, length_2, length_3) sentence_tensor = torch.from_numpy(sentence_array).float() attention_tensor = torch.from_numpy(attention_array).float() aggregated_array = util.weighted_sum(sentence_tensor, attention_tensor).data.numpy() assert aggregated_array.shape == (batch_size, length_1, length_2, embedding_dim) for i in range(length_1): for j in range(length_2): expected_array = (attention_array[0, i, j, 0] * sentence_array[0, 0] + attention_array[0, i, j, 1] * sentence_array[0, 1]) numpy.testing.assert_almost_equal(aggregated_array[0, i, j], expected_array, decimal=5)
def _encode( self, ref_target_tokens: Dict[str, torch.Tensor], ref_source_tokens: Dict[str, torch.Tensor], instance_source_tokens: Dict[str, torch.Tensor] ) -> Dict[str, torch.Tensor]: # shape: (batch_size, max_input_sequence_length, encoder_input_dim) embedded_ref_target = self._target_embedder(ref_target_tokens) ref_target_mask = util.get_text_field_mask(ref_target_tokens) encoded_ref_target = self._target_encoder(embedded_ref_target, ref_target_mask) embedded_ref_source = self._source_embedder(ref_source_tokens) ref_source_mask = util.get_text_field_mask(ref_source_tokens) encoded_ref_source = self._source_encoder(embedded_ref_source, ref_source_mask) embedded_instance_source = self._source_embedder( instance_source_tokens) instance_source_mask = util.get_text_field_mask(instance_source_tokens) encoded_instance_source = self._source_encoder( embedded_instance_source, instance_source_mask) #reduced_encoded_ref_source = torch.sum(encoded_ref_source,dim=1).unsqueeze(1) #reduced_encoded_instance_source = torch.sum(encoded_instance_source,dim=1).unsqueeze(1) #embedding_matrix = util.weighted_sum(reduced_encoded_instance_source,reduced_encoded_ref_source.permute(0,2,1)) ref_instance_similarity = self._instance_ref_sim( encoded_ref_source, encoded_instance_source) #ref_instance_similarity = -1*ref_instance_similarity #ref_instance_attention = util.masked_softmax(ref_instance_similarity, instance_source_mask) instance_ref_vectors = util.weighted_sum(encoded_instance_source, ref_instance_similarity) #target_instance_ref_similarity = self._target_instref_sim(encoded_ref_target,instance_ref_vectors) #source_target_attention = util.masked_softmax(target_instance_ref_similarity,instance_source_mask) #target_vectors = util.weighted_sum(instance_ref_vectors,source_target_attention) #print('mask',instance_source_mask.shape) #print('out',target_vectors.shape) return { "source_mask": ref_target_mask, "encoder_outputs": encoded_ref_target, "attention_over_src": instance_ref_vectors, "attention_mask": ref_source_mask }
def _compute_attention(self, sentence_encodings: torch.Tensor, context_encodings: torch.Tensor, context_mask: torch.Tensor) -> torch.Tensor: """ Computes new sentence encodings using an attention mechanism between the original sentence encodings and some context encodings. The context encodings are not necessarily the context in the cloze task sense, but any vector over which the attention should be computed. Parameters ---------- sentence_encodings: (batch_size, num_sents, hidden_dim) The original sentence encodings context_encodings: (batch_size, num_contexts, hidden_dim) The representation of each context item context_mask: (batch_size, num_contexts) The context item mask Returns ------- The new sentence encodings: (batch_size, num_sents, hidden_dim) """ if self.attention is None or self.attention_layer is None: raise Exception( '`attention` and `attention_layer` must not be `None` to use attention' ) # shape: (batch_size, num_sents, num_context_tokens) attention_scores = self.attention(sentence_encodings, context_encodings) # shape: (batch_size, num_sents, num_context_tokens) attention_probabilities = masked_softmax(attention_scores, context_mask) # shape: (batch_size, num_sents, hidden_size) attention_context = weighted_sum(context_encodings, attention_probabilities) # Concatenate the attention context with the sentence encodings # then project back to the sentence encoder hidden size # shape: (batch_size, num_sents, hidden_size * 2) concat = torch.cat([attention_context, sentence_encodings], dim=2) # shape: (batch_size, num_sents, hidden_size) projected_hidden = self.attention_layer(concat) return projected_hidden
def attend_on_sentence(self, query, encoder_outputs, encoder_output_mask): u""" This method is almost identical to ``WikiTablesDecoderStep.attend_on_question``. We just don't return the attention weights. Given a query (which is typically the decoder hidden state), compute an attention over the output of the sentence encoder, and return a weighted sum of the sentence representations given this attention. We also return the attention weights themselves. This is a simple computation, but we have it as a separate method so that the ``forward`` method on the main parser module can call it on the initial hidden state, to simplify the logic in ``take_step``. """ # (group_size, sentence_length) sentence_attention_weights = self._input_attention( query, encoder_outputs, encoder_output_mask) # (group_size, encoder_output_dim) attended_sentence = nn_util.weighted_sum(encoder_outputs, sentence_attention_weights) return attended_sentence
def forward( self, sequence_tensor: torch.FloatTensor, span_indices: torch.LongTensor, span_indices_mask: torch.BoolTensor = None, ) -> torch.FloatTensor: # shape (batch_size, sequence_length, 1) global_attention_logits = torch.matmul( sequence_tensor, torch.zeros(self.input_dim, 1).to_device(sequence_tensor.device())) # shape (batch_size, sequence_length, embedding_dim + 1) concat_tensor = torch.cat([sequence_tensor, global_attention_logits], -1) concat_output, span_mask = util.batched_span_select( concat_tensor, span_indices) print(span_mask) # Shape: (batch_size, num_spans, max_batch_span_width, embedding_dim) span_embeddings = concat_output[:, :, :, :-1] # Shape: (batch_size, num_spans, max_batch_span_width) span_attention_logits = concat_output[:, :, :, -1] # Shape: (batch_size, num_spans, max_batch_span_width) span_attention_weights = util.masked_softmax(span_attention_logits, span_mask) # Do a weighted sum of the embedded spans with # respect to the normalised attention distributions. # Shape: (batch_size, num_spans, embedding_dim) attended_text_embeddings = util.weighted_sum(span_embeddings, span_attention_weights) if span_indices_mask is not None: # Above we were masking the widths of spans with respect to the max # span width in the batch. Here we are masking the spans which were # originally passed in as padding. return attended_text_embeddings * span_indices_mask.unsqueeze(-1) return attended_text_embeddings
def prepare_decode_step_input( self, input_indices: torch.LongTensor, decoder_hidden: torch.LongTensor, encoder_outputs: torch.LongTensor, encoder_outputs_mask: torch.LongTensor) -> torch.LongTensor: """ Prepares the current timestep input for the decoder. By default, simply embeds and returns the input. If using attention, the default attention (BiLinearAttention) is applied to attend on the step input given the encoded source sequence and the previous hidden state. Parameters: ----------- input_indices : torch.LongTensor Indices of either the gold inputs to the decoder or the predicted labels from the previous timestep. decoder_hidden : torch.LongTensor, optional (not needed if no attention) Output from the decoder at the last time step. Needed only if using attention. encoder_outputs : torch.LongTensor, optional (not needed if no attention) Encoder outputs from all time steps. Needed only if using attention. encoder_outputs_mask : torch.LongTensor, optional (not needed if no attention) Masks on encoder outputs. Needed only if using attention. """ # input_indices : (batch_size,) since we are processing these one timestep at a time. # (batch_size, target_embedding_dim) embedded_input = self.target_embedder(input_indices) if self.apply_attention: if isinstance(decoder_hidden, tuple): decoder_hidden = decoder_hidden[0] # encoder_outputs : (batch_size, input_sequence_length, encoder_output_dim) # Ensuring mask is also a FloatTensor. Or else the multiplication within attention will # complain. encoder_outputs_mask = encoder_outputs_mask.float() # (batch_size, input_sequence_length) input_weights = self.decoder_attention_function( decoder_hidden[-1], encoder_outputs, encoder_outputs_mask) # (batch_size, encoder_output_dim) attended_input = util.weighted_sum(encoder_outputs, input_weights) # (batch_size, encoder_output_dim + target_embedding_dim) return torch.cat((attended_input, embedded_input), -1) else: return embedded_input
def _prepare_decode_step_input(self, input_indices: torch.LongTensor, decoder_hidden_state: torch.LongTensor = None, encoder_outputs: torch.LongTensor = None, encoder_outputs_mask: torch.LongTensor = None) -> torch.LongTensor: """ Given the input indices for the current timestep of the decoder, and all the encoder outputs, compute the input at the current timestep. Note: This method is agnostic to whether the indices are gold indices or the predictions made by the decoder at the last timestep. So, this can be used even if we're doing some kind of scheduled sampling. If we're not using attention, the output of this method is just an embedding of the input indices. If we are, the output will be a concatentation of the embedding and an attended average of the encoder inputs. Parameters ---------- input_indices : torch.LongTensor Indices of either the gold inputs to the decoder or the predicted labels from the previous timestep. decoder_hidden_state : torch.LongTensor, optional (not needed if no attention) Output of from the decoder at the last time step. Needed only if using attention. encoder_outputs : torch.LongTensor, optional (not needed if no attention) Encoder outputs from all time steps. Needed only if using attention. encoder_outputs_mask : torch.LongTensor, optional (not needed if no attention) Masks on encoder outputs. Needed only if using attention. """ # input_indices : (batch_size,) since we are processing these one timestep at a time. # (batch_size, target_embedding_dim) embedded_input = self._target_embedder(input_indices) if self._attention_function: # encoder_outputs : (batch_size, input_sequence_length, encoder_output_dim) # Ensuring mask is also a FloatTensor. Or else the multiplication within attention will # complain. encoder_outputs_mask = encoder_outputs_mask.float() # (batch_size, input_sequence_length) input_weights = self._decoder_attention(decoder_hidden_state, encoder_outputs, encoder_outputs_mask) # (batch_size, encoder_output_dim) attended_input = weighted_sum(encoder_outputs, input_weights) # (batch_size, encoder_output_dim + target_embedding_dim) return torch.cat((attended_input, embedded_input), -1) else: return embedded_input
def attend_on_question( self, query: torch.Tensor, encoder_outputs: torch.Tensor, encoder_output_mask: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: """ Given a query (which is typically the decoder hidden state), compute an attention over the output of the question encoder, and return a weighted sum of the question representations given this attention. We also return the attention weights themselves. This is a simple computation, but we have it as a separate method so that the ``forward`` method on the main parser module can call it on the initial hidden state, to simplify the logic in ``take_step``. """ # (group_size, question_length) question_attention_weights = self._input_attention( query, encoder_outputs, encoder_output_mask ) # (group_size, encoder_output_dim) attended_question = util.weighted_sum(encoder_outputs, question_attention_weights) return attended_question, question_attention_weights
def _prepare_output_projections( self, last_predictions: torch.Tensor, state: Dict[str, torch.Tensor] ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: # shape: (group_size, max_input_sequence_length, encoder_output_dim) encoder_outputs = state["encoder_outputs"] # shape: (group_size, max_input_sequence_length) source_mask = state["source_mask"] # shape: (group_size, decoder_output_dim) decoder_hidden = state["decoder_hidden"] # shape: (group_size, decoder_output_dim) decoder_context = state["decoder_context"] is_unk = (last_predictions >= self._target_vocab_size).long() last_predictions_fixed = last_predictions - last_predictions * is_unk + self._target_unk_index * is_unk embedded_input = self._target_embedder.forward(last_predictions_fixed) if not self._use_coverage: attn_scores = self._attention.forward(decoder_hidden, encoder_outputs, source_mask) else: coverage = state["coverage"] attn_scores = self._attention.forward(decoder_hidden, encoder_outputs, source_mask, coverage) coverage = coverage + attn_scores state["coverage"] = coverage attn_context = util.weighted_sum(encoder_outputs, attn_scores) decoder_input = torch.cat((attn_context, embedded_input), -1) decoder_hidden, decoder_context = self._decoder_cell( decoder_input, (decoder_hidden, decoder_context)) output_projections = self._output_projection_layer( self._hidden_projection_layer(decoder_hidden)) state["decoder_input"] = decoder_input state["decoder_hidden"] = decoder_hidden state["decoder_context"] = decoder_context state["attn_scores"] = attn_scores state["attn_context"] = attn_context return output_projections, state
def forward(self, tokens: torch.Tensor, mask: torch.Tensor): # pylint: disable=arguments-differ batch_size, sequence_length, _ = tokens.size() # Shape: (batch_size, sequence_length, sequence_length) similarity_matrix = self._matrix_attention(tokens, tokens) if self._num_attention_heads > 1: # In this case, the similarity matrix actually has shape # (batch_size, sequence_length, sequence_length, num_heads). To make the rest of the # logic below easier, we'll permute this to # (batch_size, sequence_length, num_heads, sequence_length). similarity_matrix = similarity_matrix.permute(0, 1, 3, 2) # Shape: (batch_size, sequence_length, [num_heads,] sequence_length) intra_sentence_attention = util.masked_softmax(similarity_matrix.contiguous(), mask) # Shape: (batch_size, sequence_length, projection_dim) output_token_representation = self._projection(tokens) if self._num_attention_heads > 1: # We need to split and permute the output representation to be # (batch_size, num_heads, sequence_length, projection_dim / num_heads), so that we can # do a proper weighted sum with `intra_sentence_attention`. shape = list(output_token_representation.size()) new_shape = shape[:-1] + [self._num_attention_heads, -1] # Shape: (batch_size, sequence_length, num_heads, projection_dim / num_heads) output_token_representation = output_token_representation.view(*new_shape) # Shape: (batch_size, num_heads, sequence_length, projection_dim / num_heads) output_token_representation = output_token_representation.permute(0, 2, 1, 3) # Shape: (batch_size, sequence_length, [num_heads,] projection_dim [/ num_heads]) attended_sentence = util.weighted_sum(output_token_representation, intra_sentence_attention) if self._num_attention_heads > 1: # Here we concatenate the weighted representation for each head. We'll accomplish this # just with a resize. # Shape: (batch_size, sequence_length, projection_dim) attended_sentence = attended_sentence.view(batch_size, sequence_length, -1) # Shape: (batch_size, sequence_length, combination_dim) combined_tensors = util.combine_tensors(self._combination, [tokens, attended_sentence]) return self._output_projection(combined_tensors)
def attend_on_question(self, query: torch.Tensor, encoder_outputs: torch.Tensor, encoder_output_mask: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: """ Given a query (which is typically the decoder hidden state), compute an attention over the output of the question encoder, and return a weighted sum of the question representations given this attention. We also return the attention weights themselves. This is a simple computation, but we have it as a separate method so that the ``forward`` method on the main parser module can call it on the initial hidden state, to simplify the logic in ``take_step``. """ # (group_size, question_length) question_attention_weights = self._input_attention(query, encoder_outputs, encoder_output_mask) # (group_size, encoder_output_dim) attended_question = util.weighted_sum(encoder_outputs, question_attention_weights) return attended_question, question_attention_weights
def _decode_step_output( self, decoder_hidden_state: torch.LongTensor = None, encoder_outputs: torch.LongTensor = None, encoder_outputs_mask: torch.LongTensor = None) -> torch.LongTensor: # encoder_outputs : (batch_size, input_sequence_length, encoder_output_dim) # Ensuring mask is also a FloatTensor. Or else the multiplication within attention will # complain. encoder_outputs_mask = encoder_outputs_mask.float() # (batch_size, input_sequence_length) input_weights_e = self._decoder_attention(decoder_hidden_state, encoder_outputs, encoder_outputs_mask) input_weights_a = F.softmax(input_weights_e, dim=-1) # (batch_size, encoder_output_dim) attended_input = weighted_sum(encoder_outputs, input_weights_a) #H*_t = sum(h_i*at_i) # (batch_size, encoder_output_dim + decoder_hidden_dim) return input_weights_e, input_weights_a, torch.cat( (decoder_hidden_state, attended_input), -1)
def filter(self, objects: ObjectSet, question_attention: torch.Tensor) -> ObjectSet: if self.nmn_settings["mask_non_attention"]: language_encoding = self.language_encoding[ self.object_scores_index] visual_encoding = self.visual_encoding[self.object_scores_index] cross_encoding = self.cross_encoding[self.object_scores_index] else: language_encoding = self.language_encoding visual_encoding = self.visual_encoding cross_encoding = self.cross_encoding question_attention = torch.nn.functional.pad( question_attention.view(-1), pad=(0, language_encoding.shape[0] - question_attention.numel()), ) attended_question = util.weighted_sum(language_encoding, question_attention) attended_question = attended_question.view(1, -1).repeat( visual_encoding.shape[0], 1) attended_objects = visual_encoding if self.nmn_settings["filter_find_same_params"]: attended_objects = visual_encoding.view(-1, visual_encoding.shape[-1]) find_layer_inputs = torch.cat( (attended_objects, attended_question), dim=-1) logits = self.parameters.find_layer(find_layer_inputs) filter_probs = torch.sigmoid(logits).view(-1) elif self.nmn_settings["use_sum_counting"]: attended_objects = visual_encoding logits = self.parameters.filter_layer( torch.cat( ( visual_encoding.view(-1, visual_encoding.shape[-1]), attended_question.repeat_interleave( visual_encoding.shape[1], dim=0), ), dim=-1, )) filter_probs = torch.sigmoid(logits).view(-1) self.object_scores[self.object_scores_index] = filter_probs return filter_probs * objects
def _prepare_attended_input( self, decoder_hidden_state: torch.LongTensor = None, encoder_outputs: torch.LongTensor = None, encoder_outputs_mask: torch.LongTensor = None ) -> Tuple[torch.Tensor, torch.Tensor]: """Apply attention over encoder outputs and decoder state.""" # Ensure mask is also a FloatTensor. Or else the multiplication within # attention will complain. # shape: (batch_size, max_input_sequence_length, encoder_output_dim) encoder_outputs_mask = encoder_outputs_mask.float() # shape: (batch_size, max_input_sequence_length) input_logits = self._attention(decoder_hidden_state, encoder_outputs, encoder_outputs_mask) # the attention mechanism returns the logits that are necessary for attention supervision loss, # so we normalize it here input_weights = masked_softmax(input_logits, encoder_outputs_mask) # shape: (batch_size, encoder_output_dim) attended_input = util.weighted_sum(encoder_outputs, input_weights) return attended_input, input_logits
def _forward_internal(self, text: Dict[str, torch.Tensor], relations: Dict[str, torch.Tensor]) -> torch.Tensor: t_mask = util.get_text_field_mask(text) t_emb = self.word_embeddings(text) t_hiddens = self.encoder_dropout(self.text_encoder(t_emb, t_mask)) t_encoding = self.text_attn(t_hiddens) r_masks = util.get_text_field_mask(relations, num_wrapping_dims=1) r_embs = self.word_embeddings(relations) r_sentence_encodings = self.encoder_dropout( seq_over_seq(self.relation_encoder, r_embs, r_masks)) r_attn = self.relation_attn(vector=t_encoding, matrix=r_sentence_encodings) r_encoding = util.weighted_sum(r_sentence_encodings, r_attn) final = torch.cat((t_encoding, r_encoding), dim=-1) logit = self.output(final).squeeze(-1) return cast(torch.Tensor, logit)
def with_relation(self, a: ObjectSet, b: ObjectSet, question_attention: torch.Tensor) -> ObjectSet: if self.nmn_settings["mask_non_attention"]: language_encoding = self.language_encoding[ self.object_scores_index] visual_encoding = self.visual_encoding[self.object_scores_index] cross_encoding = self.cross_encoding[self.object_scores_index] else: language_encoding = self.language_encoding visual_encoding = self.visual_encoding cross_encoding = self.cross_encoding question_attention = torch.nn.functional.pad( question_attention.view(-1), pad=(0, language_encoding.shape[0] - question_attention.numel()), ) attended_question = util.weighted_sum(language_encoding, question_attention) attended_question = attended_question.view(1, -1).repeat( visual_encoding.shape[0]**2, 1) visual_encoding_rows = visual_encoding.repeat( 1, visual_encoding.shape[0]).view(visual_encoding.shape[0]**2, -1) visual_encoding_columns = visual_encoding.repeat( visual_encoding.shape[0], 1) relate_layer_inputs = torch.cat( (visual_encoding_rows, visual_encoding_columns, attended_question), dim=-1) object_pair_scores = torch.sigmoid( self.parameters.relate_layer(relate_layer_inputs)) object_pair_scores = object_pair_scores.view(visual_encoding.shape[0], visual_encoding.shape[0]) object_pair_scores2 = object_pair_scores * ( 1.0 - torch.eye(visual_encoding.shape[0]).cuda()) object_pair_scores3 = object_pair_scores2 * a.view(-1, 1).repeat( 1, visual_encoding.shape[0]) object_pair_scores4 = object_pair_scores3 * b.view(1, -1).repeat( visual_encoding.shape[0], 1) object_scores = object_pair_scores4.sum(1).clamp( min=0.0 + self.parameters.epsilon, max=1.0 - self.parameters.epsilon) return object_scores
def _decoder_step( self, last_actions: torch.Tensor, state: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: embed_actions = self.EMB({"tokens": last_actions}) batch_size = embed_actions.size(0) # Update stack given draft pointer information draft_head = state["encoded_draft"][torch.arange(batch_size), state["draft_pointer"]] query = torch.cat([state["stream_hidden"], draft_head], dim=1) attend = self.ATTN(query, state["encoded_triple"], state["triple_mask"]) attended_triple = util.weighted_sum(state["encoded_triple"], attend) is_added = torch.stack([last_actions != tok for tok in self.SYMBOL]).all(dim=0) draft_head[is_added] = self.ADD(embed_actions[is_added]) hs, cs = self.STREAM(torch.cat((draft_head, attended_triple), dim=-1), (state["stream_hidden"], state["stream_context"])) drop_mask = (last_actions != self.DROP).unsqueeze(1).float() hx = drop_mask * hs + (-drop_mask + 1) * state["stream_hidden"] cx = drop_mask * cs + (-drop_mask + 1) * state["stream_context"] state["stream_hidden"], state["stream_context"] = hx, cx # Update Pointer move_forward = ((last_actions == self.KEEP) | (last_actions == self.DROP)).long() state["draft_pointer"] = state["draft_pointer"] + move_forward # Simple masking for pointer state["draft_pointer"] = torch.min(state["draft_pointer"], state["end_point"]) is_ended = state["end_point"] == state["draft_pointer"] state["action_mask"][is_ended, self.KEEP] = 0 state["action_mask"][is_ended, self.DROP] = 0 state["action_mask"][is_ended, self.END] = 1 return state
def summary_vector(self, encoding, mask, in_type="passage"): """ In NABERT (and in NAQANET), a 'summary_vector' is created for some entities, such as the passage or the question. This vector is created as a weighted sum of the elements of the entity, e.g. the passage summary vector is a weighted sum of the passage tokens. The specific weighting for every entity type is a learned. Parameters ---------- encoding : BERT's output layer mask : a Tensor with 1s only at the positions relevant to ``in_type`` in_type : the entity we want to summarize, e.g the passage Returns ------- The summary vector according to ``in_type``. """ if in_type == "passage": # Shape: (batch_size, seqlen) alpha = self._passage_weights_predictor(encoding).squeeze() elif in_type == "question": # Shape: (batch_size, seqlen) alpha = self._question_weights_predictor(encoding).squeeze() elif in_type == "arithmetic": # Shape: (batch_size, seqlen) alpha = self._arithmetic_weights_predictor(encoding).squeeze() elif in_type == "multiple_spans": #TODO: currenttly not using it... alpha = self._multispan_weights_predictor(encoding).squeeze() else: # Shape: (batch_size, #num of numbers, seqlen) alpha = torch.zeros(encoding.shape[:-1], device=encoding.device) # Shape: (batch_size, seqlen) # (batch_size, #num of numbers, seqlen) for numbers alpha = masked_softmax(alpha, mask) # Shape: (batch_size, out) # (batch_size, #num of numbers, out) for numbers h = util.weighted_sum(encoding, alpha) return h
def _adv_arithmetic_module(self, summary_vector, maxlen, options, options_mask, bert_out, bert_mask): # summary_vector: (batch, bert_dim) # options : (batch, opnumlen, bert_dim) # options_mask : (batch, opnumlen) # bert_out : (batch, seqlen, bert_dim) # bert_mask: (batch, seqlen) # summary_vector : (batch, explen, bert_dim) summary_vector = summary_vector.unsqueeze(1).expand(-1, maxlen, -1) # out: (batch, explen, rnn_hdim) out, _ = self.rnn(summary_vector) out = self.rnndropout(out) out = self.Wst(out) # alpha : (batch, explen, seqlen) alpha = torch.bmm(out, bert_out.transpose(1, 2)) alpha = util.masked_softmax(alpha, bert_mask) # context : (batch, explen, bert_dim) context = util.weighted_sum(bert_out, alpha) # context = self.Wcon(context) # logits : (batch, explen, opnumlen) logits = torch.bmm(context, options.transpose(1, 2)) logits = util.replace_masked_values( logits, options_mask.unsqueeze(1).expand_as(logits), -1e7) number_mask = options_mask.clone() number_mask[:, :self.num_ops] = 0 op_mask = options_mask.clone() op_mask[:, self.num_ops:] = 0 best_expression = beam_search(self.arithmetic_K, logits.softmax(-1),\ number_mask, op_mask, self.END, self.num_ops) return logits, best_expression[0]
def summary_vector(self, encoding, mask, in_type="passage"): if in_type == "passage": # Shape: (batch_size, seqlen) alpha = self._passage_weights_predictor(encoding).squeeze() elif in_type == "question": # Shape: (batch_size, seqlen) alpha = self._question_weights_predictor(encoding).squeeze() elif in_type == "arithmetic": # Shape: (batch_size, seqlen) alpha = self._arithmetic_weights_predictor(encoding).squeeze() else: # Shape: (batch_size, #num of numbers, seqlen) alpha = torch.zeros(encoding.shape[:-1], device=encoding.device) if self.number_rep == 'attention': alpha = self._number_weights_predictor(encoding).squeeze() # Shape: (batch_size, seqlen) # (batch_size, #num of numbers, seqlen) for numbers alpha = masked_softmax(alpha, mask) # Shape: (batch_size, out) # (batch_size, #num of numbers, out) for numbers h = util.weighted_sum(encoding, alpha) return h
def test_weighted_sum_handles_higher_order_input(self): batch_size = 1 length_1 = 5 length_2 = 6 length_3 = 2 embedding_dim = 4 sentence_array = numpy.random.rand(batch_size, length_1, length_2, length_3, embedding_dim) attention_array = numpy.random.rand(batch_size, length_1, length_2, length_3) sentence_tensor = Variable(torch.from_numpy(sentence_array).float()) attention_tensor = Variable(torch.from_numpy(attention_array).float()) aggregated_array = weighted_sum(sentence_tensor, attention_tensor).data.numpy() assert aggregated_array.shape == (batch_size, length_1, length_2, embedding_dim) expected_array = ( attention_array[0, 3, 2, 0] * sentence_array[0, 3, 2, 0] + attention_array[0, 3, 2, 1] * sentence_array[0, 3, 2, 1]) numpy.testing.assert_almost_equal(aggregated_array[0, 3, 2], expected_array, decimal=5)
def _prepare_output_projections( self, last_predictions: torch.Tensor, state: Dict[str, torch.Tensor], his_sym: torch.Tensor ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: encoder_outputs = state[ "encoder_outputs"] # bs, seq_len, encoder_output_dim source_mask = state["source_mask"] # bs * seq_len decoder_hidden = state["decoder_hidden"] # bs, decoder_output_dim decoder_context = state["decoder_context"] # bs * decoder_output embedded_input = self._target_embedder( last_predictions) # bs * target_embedding decoder_input = torch.cat((embedded_input, state['context_hidden']), -1) if self._attention: # 如果加了seq_to_seq attention input_weights = self._attention( decoder_hidden, encoder_outputs, source_mask.float()) # bs * seq_len attended_input = util.weighted_sum( encoder_outputs, input_weights) # bs * encoder_output decoder_input = torch.cat( (attended_input, embedded_input), -1) # bs * (decoder_output + target_embedding) decoder_hidden, decoder_context = self._decoder_cell( decoder_input, (decoder_hidden, decoder_context)) state["decoder_hidden"] = decoder_hidden # bs * hidden state["decoder_context"] = decoder_context # output_projections = self._output_projection_layer(torch.cat((decoder_hidden,graph_hidden),-1)) output_projections = self._output_projection_layer(decoder_hidden) # sz = output_projections.size(0) # for b in range(sz): # for k,li in enumerate(self.idx_to_vocab_list): # if his_sym[b][k].item() == 1: # output_projections[b][li] = 1e-9 return output_projections, state
def forward_span(self, ds_name, dialog_repr, repeated_ds_embeddings, context_masks, span_labels=None, spans_start = None, spans_end = None): batch_size, max_dialog_len = context_masks.size() ds_dialog_sim = self._ds_dialog_attention(self._dropout(repeated_ds_embeddings), self._dropout(dialog_repr)) ds_dialog_att = util.masked_softmax(ds_dialog_sim.view(-1, max_dialog_len), context_masks.view(-1, max_dialog_len)) ds_dialog_att = ds_dialog_att.view(batch_size, max_dialog_len) ds_dialog_repr = util.weighted_sum(dialog_repr, ds_dialog_att) ds_dialog_repr = ds_dialog_repr + repeated_ds_embeddings.squeeze(1) span_label_logits = self._span_label_predictor(F.relu(self._dropout(ds_dialog_repr))) span_label_prediction = torch.argmax(span_label_logits, dim=1) span_label_loss = 0.0 if span_labels is not None: span_label_loss = self._cross_entropy(span_label_logits, span_labels) # loss averaged by #turn self._accuracy.span_label_acc(ds_name, span_label_logits, span_labels, span_labels != -1) loss = span_label_loss w = self._span_prediction_layer(self._dropout(ds_dialog_repr)).unsqueeze(1) span_start_repr = self._span_start_encoder(self._dropout(dialog_repr)) span_start_logits = torch.bmm(w, span_start_repr.transpose(1,2)).squeeze(1) span_start_probs = util.masked_softmax(span_start_logits, context_masks) span_start_logits = util.replace_masked_values(span_start_logits, context_masks.to(dtype=torch.int8), -1e7) span_end_repr = self._span_end_encoder(self._dropout(span_start_repr)) span_end_logits = torch.bmm(w, span_end_repr.transpose(1,2)).squeeze(1) span_end_probs = util.masked_softmax(span_end_logits, context_masks) span_end_logits = util.replace_masked_values(span_end_logits, context_masks.to(dtype=torch.int8), -1e7) best_span = self.get_best_span(span_start_logits, span_end_logits) best_span = best_span.view(batch_size, -1) spans_loss = 0.0 if spans_start is not None: spans_loss = self._cross_entropy(span_start_logits, spans_start) self._accuracy.span_start_acc(ds_name, span_start_logits, spans_start, spans_start != -1) spans_loss += self._cross_entropy(span_end_logits, spans_end) self._accuracy.span_end_acc(ds_name, span_end_logits, spans_end, spans_end != -1) loss += spans_loss return loss, (span_label_prediction, best_span)
def forward(self, tokens: torch.Tensor, label: torch.Tensor = None) -> Dict[str, torch.Tensor]: mask = get_text_field_mask(tokens) embedding = self._word_embedding(tokens) hidden = self._encoder(embedding, mask) attention = self._attention(hidden, mask) logit = self._output(weighted_sum(hidden, attention)) output_dict: Dict[str, torch.Tensor] = { "logit": logit, "attention": attention } if label is not None: output_dict["loss"] = self._loss(logit, label) for eval_metric in self._metrics.values(): eval_metric(logit, label) return output_dict
def _decode_step_output( self, decoder_hidden_state: torch.LongTensor = None, encoder_outputs: torch.LongTensor = None, encoder_outputs_mask: torch.LongTensor = None) -> torch.LongTensor: encoder_outputs_mask = encoder_outputs_mask.float() #decribe("encoder_outputs_mask", encoder_outputs_mask) input_weights_e = self._decoder_attention(decoder_hidden_state, encoder_outputs, encoder_outputs_mask) #decribe("input_weights_e", input_weights_e) input_weights_a = masked_softmax(input_weights_e, encoder_outputs_mask) #decribe("input_weights_a", input_weights_a) attended_input = weighted_sum(encoder_outputs, input_weights_a) #decribe("attended_input", attended_input) return input_weights_a, torch.cat( (decoder_hidden_state, attended_input), -1)
similarity_function = LinearSimilarity( combination = "x,y,x*y", tensor_1_dim = 200, tensor_2_dim = 200) matrix_attention = LegacyMatrixAttention(similarity_function) passage_question_similarity = matrix_attention(encoded_passage, encoded_question) # Shape: (batch_size, passage_length, question_length) print ("passage question similarity: ", passage_question_similarity.shape) # Shape: (batch_size, passage_length, question_length) passage_question_attention = util.masked_softmax(passage_question_similarity, question_mask) # Shape: (batch_size, passage_length, encoding_dim) passage_question_vectors = util.weighted_sum(encoded_question, passage_question_attention) # We replace masked values with something really negative here, so they don't affect the # max below. masked_similarity = util.replace_masked_values(passage_question_similarity, question_mask.unsqueeze(1), -1e7) # Shape: (batch_size, passage_length) question_passage_similarity = masked_similarity.max(dim=-1)[0].squeeze(-1) # Shape: (batch_size, passage_length) question_passage_attention = util.masked_softmax(question_passage_similarity, passage_mask) # Shape: (batch_size, encoding_dim) question_passage_vector = util.weighted_sum(encoded_passage, question_passage_attention) # Shape: (batch_size, passage_length, encoding_dim) tiled_question_passage_vector = question_passage_vector.unsqueeze(1).expand(batch_size, passage_length,
def _get_initial_rnn_and_grammar_state(self, question: Dict[str, torch.LongTensor], table: Dict[str, torch.LongTensor], world: List[WikiTablesWorld], actions: List[List[ProductionRule]], outputs: Dict[str, Any]) -> Tuple[List[RnnStatelet], List[LambdaGrammarStatelet]]: """ Encodes the question and table, computes a linking between the two, and constructs an initial RnnStatelet and LambdaGrammarStatelet for each batch instance to pass to the decoder. We take ``outputs`` as a parameter here and `modify` it, adding things that we want to visualize in a demo. """ table_text = table['text'] # (batch_size, question_length, embedding_dim) embedded_question = self._question_embedder(question) question_mask = util.get_text_field_mask(question).float() # (batch_size, num_entities, num_entity_tokens, embedding_dim) embedded_table = self._question_embedder(table_text, num_wrapping_dims=1) table_mask = util.get_text_field_mask(table_text, num_wrapping_dims=1).float() batch_size, num_entities, num_entity_tokens, _ = embedded_table.size() num_question_tokens = embedded_question.size(1) # (batch_size, num_entities, embedding_dim) encoded_table = self._entity_encoder(embedded_table, table_mask) # (batch_size, num_entities, num_neighbors) neighbor_indices = self._get_neighbor_indices(world, num_entities, encoded_table) # Neighbor_indices is padded with -1 since 0 is a potential neighbor index. # Thus, the absolute value needs to be taken in the index_select, and 1 needs to # be added for the mask since that method expects 0 for padding. # (batch_size, num_entities, num_neighbors, embedding_dim) embedded_neighbors = util.batched_index_select(encoded_table, torch.abs(neighbor_indices)) neighbor_mask = util.get_text_field_mask({'ignored': neighbor_indices + 1}, num_wrapping_dims=1).float() # Encoder initialized to easily obtain a masked average. neighbor_encoder = TimeDistributed(BagOfEmbeddingsEncoder(self._embedding_dim, averaged=True)) # (batch_size, num_entities, embedding_dim) embedded_neighbors = neighbor_encoder(embedded_neighbors, neighbor_mask) # entity_types: tensor with shape (batch_size, num_entities), where each entry is the # entity's type id. # entity_type_dict: Dict[int, int], mapping flattened_entity_index -> type_index # These encode the same information, but for efficiency reasons later it's nice # to have one version as a tensor and one that's accessible on the cpu. entity_types, entity_type_dict = self._get_type_vector(world, num_entities, encoded_table) entity_type_embeddings = self._entity_type_encoder_embedding(entity_types) projected_neighbor_embeddings = self._neighbor_params(embedded_neighbors.float()) # (batch_size, num_entities, embedding_dim) entity_embeddings = torch.tanh(entity_type_embeddings + projected_neighbor_embeddings) # Compute entity and question word similarity. We tried using cosine distance here, but # because this similarity is the main mechanism that the model can use to push apart logit # scores for certain actions (like "n -> 1" and "n -> -1"), this needs to have a larger # output range than [-1, 1]. question_entity_similarity = torch.bmm(embedded_table.view(batch_size, num_entities * num_entity_tokens, self._embedding_dim), torch.transpose(embedded_question, 1, 2)) question_entity_similarity = question_entity_similarity.view(batch_size, num_entities, num_entity_tokens, num_question_tokens) # (batch_size, num_entities, num_question_tokens) question_entity_similarity_max_score, _ = torch.max(question_entity_similarity, 2) # (batch_size, num_entities, num_question_tokens, num_features) linking_features = table['linking'] linking_scores = question_entity_similarity_max_score if self._use_neighbor_similarity_for_linking: # The linking score is computed as a linear projection of two terms. The first is the # maximum similarity score over the entity's words and the question token. The second # is the maximum similarity over the words in the entity's neighbors and the question # token. # # The second term, projected_question_neighbor_similarity, is useful when a column # needs to be selected. For example, the question token might have no similarity with # the column name, but is similar with the cells in the column. # # Note that projected_question_neighbor_similarity is intended to capture the same # information as the related_column feature. # # Also note that this block needs to be _before_ the `linking_params` block, because # we're overwriting `linking_scores`, not adding to it. # (batch_size, num_entities, num_neighbors, num_question_tokens) question_neighbor_similarity = util.batched_index_select(question_entity_similarity_max_score, torch.abs(neighbor_indices)) # (batch_size, num_entities, num_question_tokens) question_neighbor_similarity_max_score, _ = torch.max(question_neighbor_similarity, 2) projected_question_entity_similarity = self._question_entity_params( question_entity_similarity_max_score.unsqueeze(-1)).squeeze(-1) projected_question_neighbor_similarity = self._question_neighbor_params( question_neighbor_similarity_max_score.unsqueeze(-1)).squeeze(-1) linking_scores = projected_question_entity_similarity + projected_question_neighbor_similarity feature_scores = None if self._linking_params is not None: feature_scores = self._linking_params(linking_features).squeeze(3) linking_scores = linking_scores + feature_scores # (batch_size, num_question_tokens, num_entities) linking_probabilities = self._get_linking_probabilities(world, linking_scores.transpose(1, 2), question_mask, entity_type_dict) # (batch_size, num_question_tokens, embedding_dim) link_embedding = util.weighted_sum(entity_embeddings, linking_probabilities) encoder_input = torch.cat([link_embedding, embedded_question], 2) # (batch_size, question_length, encoder_output_dim) encoder_outputs = self._dropout(self._encoder(encoder_input, question_mask)) # This will be our initial hidden state and memory cell for the decoder LSTM. final_encoder_output = util.get_final_encoder_states(encoder_outputs, question_mask, self._encoder.is_bidirectional()) memory_cell = encoder_outputs.new_zeros(batch_size, self._encoder.get_output_dim()) # To make grouping states together in the decoder easier, we convert the batch dimension in # all of our tensors into an outer list. For instance, the encoder outputs have shape # `(batch_size, question_length, encoder_output_dim)`. We need to convert this into a list # of `batch_size` tensors, each of shape `(question_length, encoder_output_dim)`. Then we # won't have to do any index selects, or anything, we'll just do some `torch.cat()`s. encoder_output_list = [encoder_outputs[i] for i in range(batch_size)] question_mask_list = [question_mask[i] for i in range(batch_size)] initial_rnn_state = [] for i in range(batch_size): initial_rnn_state.append(RnnStatelet(final_encoder_output[i], memory_cell[i], self._first_action_embedding, self._first_attended_question, encoder_output_list, question_mask_list)) initial_grammar_state = [self._create_grammar_state(world[i], actions[i], linking_scores[i], entity_types[i]) for i in range(batch_size)] if not self.training: # We add a few things to the outputs that will be returned from `forward` at evaluation # time, for visualization in a demo. outputs['linking_scores'] = linking_scores if feature_scores is not None: outputs['feature_scores'] = feature_scores outputs['similarity_scores'] = question_entity_similarity_max_score return initial_rnn_state, initial_grammar_state
def forward(self, # type: ignore premise: Dict[str, torch.LongTensor], hypothesis: Dict[str, torch.LongTensor], label: torch.IntTensor = None, metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ Parameters ---------- premise : Dict[str, torch.LongTensor] From a ``TextField`` hypothesis : Dict[str, torch.LongTensor] From a ``TextField`` label : torch.IntTensor, optional, (default = None) From a ``LabelField`` metadata : ``List[Dict[str, Any]]``, optional, (default = None) Metadata containing the original tokenization of the premise and hypothesis with 'premise_tokens' and 'hypothesis_tokens' keys respectively. Returns ------- An output dictionary consisting of: label_logits : torch.FloatTensor A tensor of shape ``(batch_size, num_labels)`` representing unnormalised log probabilities of the entailment label. label_probs : torch.FloatTensor A tensor of shape ``(batch_size, num_labels)`` representing probabilities of the entailment label. loss : torch.FloatTensor, optional A scalar loss to be optimised. """ embedded_premise = self._text_field_embedder(premise) embedded_hypothesis = self._text_field_embedder(hypothesis) premise_mask = get_text_field_mask(premise).float() hypothesis_mask = get_text_field_mask(hypothesis).float() if self._premise_encoder: embedded_premise = self._premise_encoder(embedded_premise, premise_mask) if self._hypothesis_encoder: embedded_hypothesis = self._hypothesis_encoder(embedded_hypothesis, hypothesis_mask) projected_premise = self._attend_feedforward(embedded_premise) projected_hypothesis = self._attend_feedforward(embedded_hypothesis) # Shape: (batch_size, premise_length, hypothesis_length) similarity_matrix = self._matrix_attention(projected_premise, projected_hypothesis) # Shape: (batch_size, premise_length, hypothesis_length) p2h_attention = masked_softmax(similarity_matrix, hypothesis_mask) # Shape: (batch_size, premise_length, embedding_dim) attended_hypothesis = weighted_sum(embedded_hypothesis, p2h_attention) # Shape: (batch_size, hypothesis_length, premise_length) h2p_attention = masked_softmax(similarity_matrix.transpose(1, 2).contiguous(), premise_mask) # Shape: (batch_size, hypothesis_length, embedding_dim) attended_premise = weighted_sum(embedded_premise, h2p_attention) premise_compare_input = torch.cat([embedded_premise, attended_hypothesis], dim=-1) hypothesis_compare_input = torch.cat([embedded_hypothesis, attended_premise], dim=-1) compared_premise = self._compare_feedforward(premise_compare_input) compared_premise = compared_premise * premise_mask.unsqueeze(-1) # Shape: (batch_size, compare_dim) compared_premise = compared_premise.sum(dim=1) compared_hypothesis = self._compare_feedforward(hypothesis_compare_input) compared_hypothesis = compared_hypothesis * hypothesis_mask.unsqueeze(-1) # Shape: (batch_size, compare_dim) compared_hypothesis = compared_hypothesis.sum(dim=1) aggregate_input = torch.cat([compared_premise, compared_hypothesis], dim=-1) label_logits = self._aggregate_feedforward(aggregate_input) label_probs = torch.nn.functional.softmax(label_logits, dim=-1) output_dict = {"label_logits": label_logits, "label_probs": label_probs, "h2p_attention": h2p_attention, "p2h_attention": p2h_attention} if label is not None: loss = self._loss(label_logits, label.long().view(-1)) self._accuracy(label_logits, label) output_dict["loss"] = loss if metadata is not None: output_dict["premise_tokens"] = [x["premise_tokens"] for x in metadata] output_dict["hypothesis_tokens"] = [x["hypothesis_tokens"] for x in metadata] return output_dict
def _get_initial_state_and_scores(self, question: Dict[str, torch.LongTensor], table: Dict[str, torch.LongTensor], world: List[WikiTablesWorld], actions: List[List[ProductionRuleArray]], example_lisp_string: List[str] = None, add_world_to_initial_state: bool = False, checklist_states: List[ChecklistState] = None) -> Dict: """ Does initial preparation and creates an intiial state for both the semantic parsers. Note that the checklist state is optional, and the ``WikiTablesMmlParser`` is not expected to pass it. """ table_text = table['text'] # (batch_size, question_length, embedding_dim) embedded_question = self._question_embedder(question) question_mask = util.get_text_field_mask(question).float() # (batch_size, num_entities, num_entity_tokens, embedding_dim) embedded_table = self._question_embedder(table_text, num_wrapping_dims=1) table_mask = util.get_text_field_mask(table_text, num_wrapping_dims=1).float() batch_size, num_entities, num_entity_tokens, _ = embedded_table.size() num_question_tokens = embedded_question.size(1) # (batch_size, num_entities, embedding_dim) encoded_table = self._entity_encoder(embedded_table, table_mask) # (batch_size, num_entities, num_neighbors) neighbor_indices = self._get_neighbor_indices(world, num_entities, encoded_table) # Neighbor_indices is padded with -1 since 0 is a potential neighbor index. # Thus, the absolute value needs to be taken in the index_select, and 1 needs to # be added for the mask since that method expects 0 for padding. # (batch_size, num_entities, num_neighbors, embedding_dim) embedded_neighbors = util.batched_index_select(encoded_table, torch.abs(neighbor_indices)) neighbor_mask = util.get_text_field_mask({'ignored': neighbor_indices + 1}, num_wrapping_dims=1).float() # Encoder initialized to easily obtain a masked average. neighbor_encoder = TimeDistributed(BagOfEmbeddingsEncoder(self._embedding_dim, averaged=True)) # (batch_size, num_entities, embedding_dim) embedded_neighbors = neighbor_encoder(embedded_neighbors, neighbor_mask) # entity_types: one-hot tensor with shape (batch_size, num_entities, num_types) # entity_type_dict: Dict[int, int], mapping flattened_entity_index -> type_index # These encode the same information, but for efficiency reasons later it's nice # to have one version as a tensor and one that's accessible on the cpu. entity_types, entity_type_dict = self._get_type_vector(world, num_entities, encoded_table) entity_type_embeddings = self._type_params(entity_types.float()) projected_neighbor_embeddings = self._neighbor_params(embedded_neighbors.float()) # (batch_size, num_entities, embedding_dim) entity_embeddings = torch.nn.functional.tanh(entity_type_embeddings + projected_neighbor_embeddings) # Compute entity and question word similarity. We tried using cosine distance here, but # because this similarity is the main mechanism that the model can use to push apart logit # scores for certain actions (like "n -> 1" and "n -> -1"), this needs to have a larger # output range than [-1, 1]. question_entity_similarity = torch.bmm(embedded_table.view(batch_size, num_entities * num_entity_tokens, self._embedding_dim), torch.transpose(embedded_question, 1, 2)) question_entity_similarity = question_entity_similarity.view(batch_size, num_entities, num_entity_tokens, num_question_tokens) # (batch_size, num_entities, num_question_tokens) question_entity_similarity_max_score, _ = torch.max(question_entity_similarity, 2) # (batch_size, num_entities, num_question_tokens, num_features) linking_features = table['linking'] linking_scores = question_entity_similarity_max_score if self._use_neighbor_similarity_for_linking: # The linking score is computed as a linear projection of two terms. The first is the # maximum similarity score over the entity's words and the question token. The second # is the maximum similarity over the words in the entity's neighbors and the question # token. # # The second term, projected_question_neighbor_similarity, is useful when a column # needs to be selected. For example, the question token might have no similarity with # the column name, but is similar with the cells in the column. # # Note that projected_question_neighbor_similarity is intended to capture the same # information as the related_column feature. # # Also note that this block needs to be _before_ the `linking_params` block, because # we're overwriting `linking_scores`, not adding to it. # (batch_size, num_entities, num_neighbors, num_question_tokens) question_neighbor_similarity = util.batched_index_select(question_entity_similarity_max_score, torch.abs(neighbor_indices)) # (batch_size, num_entities, num_question_tokens) question_neighbor_similarity_max_score, _ = torch.max(question_neighbor_similarity, 2) projected_question_entity_similarity = self._question_entity_params( question_entity_similarity_max_score.unsqueeze(-1)).squeeze(-1) projected_question_neighbor_similarity = self._question_neighbor_params( question_neighbor_similarity_max_score.unsqueeze(-1)).squeeze(-1) linking_scores = projected_question_entity_similarity + projected_question_neighbor_similarity feature_scores = None if self._linking_params is not None: feature_scores = self._linking_params(linking_features).squeeze(3) linking_scores = linking_scores + feature_scores # (batch_size, num_question_tokens, num_entities) linking_probabilities = self._get_linking_probabilities(world, linking_scores.transpose(1, 2), question_mask, entity_type_dict) # (batch_size, num_question_tokens, embedding_dim) link_embedding = util.weighted_sum(entity_embeddings, linking_probabilities) encoder_input = torch.cat([link_embedding, embedded_question], 2) # (batch_size, question_length, encoder_output_dim) encoder_outputs = self._dropout(self._encoder(encoder_input, question_mask)) # This will be our initial hidden state and memory cell for the decoder LSTM. final_encoder_output = util.get_final_encoder_states(encoder_outputs, question_mask, self._encoder.is_bidirectional()) memory_cell = encoder_outputs.new_zeros(batch_size, self._encoder.get_output_dim()) initial_score = embedded_question.data.new_zeros(batch_size) action_embeddings, output_action_embeddings, action_biases, action_indices = self._embed_actions(actions) _, num_entities, num_question_tokens = linking_scores.size() flattened_linking_scores, actions_to_entities = self._map_entity_productions(linking_scores, world, actions) # To make grouping states together in the decoder easier, we convert the batch dimension in # all of our tensors into an outer list. For instance, the encoder outputs have shape # `(batch_size, question_length, encoder_output_dim)`. We need to convert this into a list # of `batch_size` tensors, each of shape `(question_length, encoder_output_dim)`. Then we # won't have to do any index selects, or anything, we'll just do some `torch.cat()`s. initial_score_list = [initial_score[i] for i in range(batch_size)] encoder_output_list = [encoder_outputs[i] for i in range(batch_size)] question_mask_list = [question_mask[i] for i in range(batch_size)] initial_rnn_state = [] for i in range(batch_size): initial_rnn_state.append(RnnState(final_encoder_output[i], memory_cell[i], self._first_action_embedding, self._first_attended_question, encoder_output_list, question_mask_list)) initial_grammar_state = [self._create_grammar_state(world[i], actions[i]) for i in range(batch_size)] initial_state_world = world if add_world_to_initial_state else None initial_state = WikiTablesDecoderState(batch_indices=list(range(batch_size)), action_history=[[] for _ in range(batch_size)], score=initial_score_list, rnn_state=initial_rnn_state, grammar_state=initial_grammar_state, action_embeddings=action_embeddings, output_action_embeddings=output_action_embeddings, action_biases=action_biases, action_indices=action_indices, possible_actions=actions, flattened_linking_scores=flattened_linking_scores, actions_to_entities=actions_to_entities, entity_types=entity_type_dict, world=initial_state_world, example_lisp_string=example_lisp_string, checklist_state=checklist_states, debug_info=None) return {"initial_state": initial_state, "linking_scores": linking_scores, "feature_scores": feature_scores, "similarity_scores": question_entity_similarity_max_score}
def forward(self, sequence_tensor: torch.FloatTensor, span_indices: torch.LongTensor, sequence_mask: torch.LongTensor = None, span_indices_mask: torch.LongTensor = None) -> torch.FloatTensor: # both of shape (batch_size, num_spans, 1) span_starts, span_ends = span_indices.split(1, dim=-1) # shape (batch_size, num_spans, 1) # These span widths are off by 1, because the span ends are `inclusive`. span_widths = span_ends - span_starts # We need to know the maximum span width so we can # generate indices to extract the spans from the sequence tensor. # These indices will then get masked below, such that if the length # of a given span is smaller than the max, the rest of the values # are masked. max_batch_span_width = span_widths.max().item() + 1 # shape (batch_size, sequence_length, 1) global_attention_logits = self._global_attention(sequence_tensor) # Shape: (1, 1, max_batch_span_width) max_span_range_indices = util.get_range_vector(max_batch_span_width, util.get_device_of(sequence_tensor)).view(1, 1, -1) # Shape: (batch_size, num_spans, max_batch_span_width) # This is a broadcasted comparison - for each span we are considering, # we are creating a range vector of size max_span_width, but masking values # which are greater than the actual length of the span. # # We're using <= here (and for the mask below) because the span ends are # inclusive, so we want to include indices which are equal to span_widths rather # than using it as a non-inclusive upper bound. span_mask = (max_span_range_indices <= span_widths).float() raw_span_indices = span_ends - max_span_range_indices # We also don't want to include span indices which are less than zero, # which happens because some spans near the beginning of the sequence # have an end index < max_batch_span_width, so we add this to the mask here. span_mask = span_mask * (raw_span_indices >= 0).float() span_indices = torch.nn.functional.relu(raw_span_indices.float()).long() # Shape: (batch_size * num_spans * max_batch_span_width) flat_span_indices = util.flatten_and_batch_shift_indices(span_indices, sequence_tensor.size(1)) # Shape: (batch_size, num_spans, max_batch_span_width, embedding_dim) span_embeddings = util.batched_index_select(sequence_tensor, span_indices, flat_span_indices) # Shape: (batch_size, num_spans, max_batch_span_width) span_attention_logits = util.batched_index_select(global_attention_logits, span_indices, flat_span_indices).squeeze(-1) # Shape: (batch_size, num_spans, max_batch_span_width) span_attention_weights = util.masked_softmax(span_attention_logits, span_mask) # Do a weighted sum of the embedded spans with # respect to the normalised attention distributions. # Shape: (batch_size, num_spans, embedding_dim) attended_text_embeddings = util.weighted_sum(span_embeddings, span_attention_weights) if span_indices_mask is not None: # Above we were masking the widths of spans with respect to the max # span width in the batch. Here we are masking the spans which were # originally passed in as padding. return attended_text_embeddings * span_indices_mask.unsqueeze(-1).float() return attended_text_embeddings
def forward(self, # type: ignore premise: Dict[str, torch.LongTensor], hypothesis: Dict[str, torch.LongTensor], label: torch.IntTensor = None, metadata: List[Dict[str, Any]] = None # pylint:disable=unused-argument ) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ Parameters ---------- premise : Dict[str, torch.LongTensor] From a ``TextField`` hypothesis : Dict[str, torch.LongTensor] From a ``TextField`` label : torch.IntTensor, optional (default = None) From a ``LabelField`` metadata : ``List[Dict[str, Any]]``, optional, (default = None) Metadata containing the original tokenization of the premise and hypothesis with 'premise_tokens' and 'hypothesis_tokens' keys respectively. Returns ------- An output dictionary consisting of: label_logits : torch.FloatTensor A tensor of shape ``(batch_size, num_labels)`` representing unnormalised log probabilities of the entailment label. label_probs : torch.FloatTensor A tensor of shape ``(batch_size, num_labels)`` representing probabilities of the entailment label. loss : torch.FloatTensor, optional A scalar loss to be optimised. """ embedded_premise = self._text_field_embedder(premise) embedded_hypothesis = self._text_field_embedder(hypothesis) premise_mask = get_text_field_mask(premise).float() hypothesis_mask = get_text_field_mask(hypothesis).float() # apply dropout for LSTM if self.rnn_input_dropout: embedded_premise = self.rnn_input_dropout(embedded_premise) embedded_hypothesis = self.rnn_input_dropout(embedded_hypothesis) # encode premise and hypothesis encoded_premise = self._encoder(embedded_premise, premise_mask) encoded_hypothesis = self._encoder(embedded_hypothesis, hypothesis_mask) # Shape: (batch_size, premise_length, hypothesis_length) similarity_matrix = self._matrix_attention(encoded_premise, encoded_hypothesis) # Shape: (batch_size, premise_length, hypothesis_length) p2h_attention = last_dim_softmax(similarity_matrix, hypothesis_mask) # Shape: (batch_size, premise_length, embedding_dim) attended_hypothesis = weighted_sum(encoded_hypothesis, p2h_attention) # Shape: (batch_size, hypothesis_length, premise_length) h2p_attention = last_dim_softmax(similarity_matrix.transpose(1, 2).contiguous(), premise_mask) # Shape: (batch_size, hypothesis_length, embedding_dim) attended_premise = weighted_sum(encoded_premise, h2p_attention) # the "enhancement" layer premise_enhanced = torch.cat( [encoded_premise, attended_hypothesis, encoded_premise - attended_hypothesis, encoded_premise * attended_hypothesis], dim=-1 ) hypothesis_enhanced = torch.cat( [encoded_hypothesis, attended_premise, encoded_hypothesis - attended_premise, encoded_hypothesis * attended_premise], dim=-1 ) # The projection layer down to the model dimension. Dropout is not applied before # projection. projected_enhanced_premise = self._projection_feedforward(premise_enhanced) projected_enhanced_hypothesis = self._projection_feedforward(hypothesis_enhanced) # Run the inference layer if self.rnn_input_dropout: projected_enhanced_premise = self.rnn_input_dropout(projected_enhanced_premise) projected_enhanced_hypothesis = self.rnn_input_dropout(projected_enhanced_hypothesis) v_ai = self._inference_encoder(projected_enhanced_premise, premise_mask) v_bi = self._inference_encoder(projected_enhanced_hypothesis, hypothesis_mask) # The pooling layer -- max and avg pooling. # (batch_size, model_dim) v_a_max, _ = replace_masked_values( v_ai, premise_mask.unsqueeze(-1), -1e7 ).max(dim=1) v_b_max, _ = replace_masked_values( v_bi, hypothesis_mask.unsqueeze(-1), -1e7 ).max(dim=1) v_a_avg = torch.sum(v_ai * premise_mask.unsqueeze(-1), dim=1) / torch.sum( premise_mask, 1, keepdim=True ) v_b_avg = torch.sum(v_bi * hypothesis_mask.unsqueeze(-1), dim=1) / torch.sum( hypothesis_mask, 1, keepdim=True ) # Now concat # (batch_size, model_dim * 2 * 4) v_all = torch.cat([v_a_avg, v_a_max, v_b_avg, v_b_max], dim=1) # the final MLP -- apply dropout to input, and MLP applies to output & hidden if self.dropout: v_all = self.dropout(v_all) output_hidden = self._output_feedforward(v_all) label_logits = self._output_logit(output_hidden) label_probs = torch.nn.functional.softmax(label_logits, dim=-1) output_dict = {"label_logits": label_logits, "label_probs": label_probs} if label is not None: loss = self._loss(label_logits, label.long().view(-1)) self._accuracy(label_logits, label) output_dict["loss"] = loss return output_dict
def forward(self, # type: ignore tokens: Dict[str, torch.LongTensor], label: torch.LongTensor = None) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ Parameters ---------- tokens : Dict[str, torch.LongTensor], required The output of ``TextField.as_array()``. label : torch.LongTensor, optional (default = None) A variable representing the label for each instance in the batch. Returns ------- An output dictionary consisting of: class_probabilities : torch.FloatTensor A tensor of shape ``(batch_size, num_classes)`` representing a distribution over the label classes for each instance. loss : torch.FloatTensor, optional A scalar loss to be optimised. """ text_mask = util.get_text_field_mask(tokens).float() # Pop elmo tokens, since elmo embedder should not be present. elmo_tokens = tokens.pop("elmo", None) embedded_text = self._text_field_embedder(tokens) # Add the "elmo" key back to "tokens" if not None, since the tests and the # subsequent training epochs rely not being modified during forward() if elmo_tokens is not None: tokens["elmo"] = elmo_tokens # Create ELMo embeddings if applicable if self._elmo: if elmo_tokens is not None: elmo_representations = self._elmo(elmo_tokens)["elmo_representations"] # Pop from the end is more performant with list if self._use_integrator_output_elmo: integrator_output_elmo = elmo_representations.pop() if self._use_input_elmo: input_elmo = elmo_representations.pop() assert not elmo_representations else: raise ConfigurationError( "Model was built to use Elmo, but input text is not tokenized for Elmo.") if self._use_input_elmo: embedded_text = torch.cat([embedded_text, input_elmo], dim=-1) dropped_embedded_text = self._embedding_dropout(embedded_text) pre_encoded_text = self._pre_encode_feedforward(dropped_embedded_text) encoded_tokens = self._encoder(pre_encoded_text, text_mask) # Compute biattention. This is a special case since the inputs are the same. attention_logits = encoded_tokens.bmm(encoded_tokens.permute(0, 2, 1).contiguous()) attention_weights = util.last_dim_softmax(attention_logits, text_mask) encoded_text = util.weighted_sum(encoded_tokens, attention_weights) # Build the input to the integrator integrator_input = torch.cat([encoded_tokens, encoded_tokens - encoded_text, encoded_tokens * encoded_text], 2) integrated_encodings = self._integrator(integrator_input, text_mask) # Concatenate ELMo representations to integrated_encodings if specified if self._use_integrator_output_elmo: integrated_encodings = torch.cat([integrated_encodings, integrator_output_elmo], dim=-1) # Simple Pooling layers max_masked_integrated_encodings = util.replace_masked_values( integrated_encodings, text_mask.unsqueeze(2), -1e7) max_pool = torch.max(max_masked_integrated_encodings, 1)[0] min_masked_integrated_encodings = util.replace_masked_values( integrated_encodings, text_mask.unsqueeze(2), +1e7) min_pool = torch.min(min_masked_integrated_encodings, 1)[0] mean_pool = torch.sum(integrated_encodings, 1) / torch.sum(text_mask, 1, keepdim=True) # Self-attentive pooling layer # Run through linear projection. Shape: (batch_size, sequence length, 1) # Then remove the last dimension to get the proper attention shape (batch_size, sequence length). self_attentive_logits = self._self_attentive_pooling_projection( integrated_encodings).squeeze(2) self_weights = util.masked_softmax(self_attentive_logits, text_mask) self_attentive_pool = util.weighted_sum(integrated_encodings, self_weights) pooled_representations = torch.cat([max_pool, min_pool, mean_pool, self_attentive_pool], 1) pooled_representations_dropped = self._integrator_dropout(pooled_representations) logits = self._output_layer(pooled_representations_dropped) class_probabilities = F.softmax(logits, dim=-1) output_dict = {'logits': logits, 'class_probabilities': class_probabilities} if label is not None: loss = self.loss(logits, label) for metric in self.metrics.values(): metric(logits, label) output_dict["loss"] = loss return output_dict
def forward(self, # type: ignore question: Dict[str, torch.LongTensor], passage: Dict[str, torch.LongTensor], span_start: torch.IntTensor = None, span_end: torch.IntTensor = None, metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ Parameters ---------- question : Dict[str, torch.LongTensor] From a ``TextField``. passage : Dict[str, torch.LongTensor] From a ``TextField``. The model assumes that this passage contains the answer to the question, and predicts the beginning and ending positions of the answer within the passage. span_start : ``torch.IntTensor``, optional From an ``IndexField``. This is one of the things we are trying to predict - the beginning position of the answer with the passage. This is an `inclusive` token index. If this is given, we will compute a loss that gets included in the output dictionary. span_end : ``torch.IntTensor``, optional From an ``IndexField``. This is one of the things we are trying to predict - the ending position of the answer with the passage. This is an `inclusive` token index. If this is given, we will compute a loss that gets included in the output dictionary. metadata : ``List[Dict[str, Any]]``, optional If present, this should contain the question ID, original passage text, and token offsets into the passage for each instance in the batch. We use this for computing official metrics using the official SQuAD evaluation script. The length of this list should be the batch size, and each dictionary should have the keys ``id``, ``original_passage``, and ``token_offsets``. If you only want the best span string and don't care about official metrics, you can omit the ``id`` key. Returns ------- An output dictionary consisting of: span_start_logits : torch.FloatTensor A tensor of shape ``(batch_size, passage_length)`` representing unnormalized log probabilities of the span start position. span_start_probs : torch.FloatTensor The result of ``softmax(span_start_logits)``. span_end_logits : torch.FloatTensor A tensor of shape ``(batch_size, passage_length)`` representing unnormalized log probabilities of the span end position (inclusive). span_end_probs : torch.FloatTensor The result of ``softmax(span_end_logits)``. best_span : torch.IntTensor The result of a constrained inference over ``span_start_logits`` and ``span_end_logits`` to find the most probable span. Shape is ``(batch_size, 2)`` and each offset is a token index. loss : torch.FloatTensor, optional A scalar loss to be optimised. best_span_str : List[str] If sufficient metadata was provided for the instances in the batch, we also return the string from the original passage that the model thinks is the best answer to the question. """ embedded_question = self._highway_layer(self._text_field_embedder(question)) embedded_passage = self._highway_layer(self._text_field_embedder(passage)) batch_size = embedded_question.size(0) passage_length = embedded_passage.size(1) question_mask = util.get_text_field_mask(question).float() passage_mask = util.get_text_field_mask(passage).float() question_lstm_mask = question_mask if self._mask_lstms else None passage_lstm_mask = passage_mask if self._mask_lstms else None encoded_question = self._dropout(self._phrase_layer(embedded_question, question_lstm_mask)) encoded_passage = self._dropout(self._phrase_layer(embedded_passage, passage_lstm_mask)) encoding_dim = encoded_question.size(-1) # Shape: (batch_size, passage_length, question_length) passage_question_similarity = self._matrix_attention(encoded_passage, encoded_question) # Shape: (batch_size, passage_length, question_length) passage_question_attention = util.masked_softmax(passage_question_similarity, question_mask) # Shape: (batch_size, passage_length, encoding_dim) passage_question_vectors = util.weighted_sum(encoded_question, passage_question_attention) # We replace masked values with something really negative here, so they don't affect the # max below. masked_similarity = util.replace_masked_values(passage_question_similarity, question_mask.unsqueeze(1), -1e7) # Shape: (batch_size, passage_length) question_passage_similarity = masked_similarity.max(dim=-1)[0].squeeze(-1) # Shape: (batch_size, passage_length) question_passage_attention = util.masked_softmax(question_passage_similarity, passage_mask) # Shape: (batch_size, encoding_dim) question_passage_vector = util.weighted_sum(encoded_passage, question_passage_attention) # Shape: (batch_size, passage_length, encoding_dim) tiled_question_passage_vector = question_passage_vector.unsqueeze(1).expand(batch_size, passage_length, encoding_dim) # Shape: (batch_size, passage_length, encoding_dim * 4) final_merged_passage = torch.cat([encoded_passage, passage_question_vectors, encoded_passage * passage_question_vectors, encoded_passage * tiled_question_passage_vector], dim=-1) modeled_passage = self._dropout(self._modeling_layer(final_merged_passage, passage_lstm_mask)) modeling_dim = modeled_passage.size(-1) # Shape: (batch_size, passage_length, encoding_dim * 4 + modeling_dim)) span_start_input = self._dropout(torch.cat([final_merged_passage, modeled_passage], dim=-1)) # Shape: (batch_size, passage_length) span_start_logits = self._span_start_predictor(span_start_input).squeeze(-1) # Shape: (batch_size, passage_length) span_start_probs = util.masked_softmax(span_start_logits, passage_mask) # Shape: (batch_size, modeling_dim) span_start_representation = util.weighted_sum(modeled_passage, span_start_probs) # Shape: (batch_size, passage_length, modeling_dim) tiled_start_representation = span_start_representation.unsqueeze(1).expand(batch_size, passage_length, modeling_dim) # Shape: (batch_size, passage_length, encoding_dim * 4 + modeling_dim * 3) span_end_representation = torch.cat([final_merged_passage, modeled_passage, tiled_start_representation, modeled_passage * tiled_start_representation], dim=-1) # Shape: (batch_size, passage_length, encoding_dim) encoded_span_end = self._dropout(self._span_end_encoder(span_end_representation, passage_lstm_mask)) # Shape: (batch_size, passage_length, encoding_dim * 4 + span_end_encoding_dim) span_end_input = self._dropout(torch.cat([final_merged_passage, encoded_span_end], dim=-1)) span_end_logits = self._span_end_predictor(span_end_input).squeeze(-1) span_end_probs = util.masked_softmax(span_end_logits, passage_mask) span_start_logits = util.replace_masked_values(span_start_logits, passage_mask, -1e7) span_end_logits = util.replace_masked_values(span_end_logits, passage_mask, -1e7) best_span = self.get_best_span(span_start_logits, span_end_logits) output_dict = { "passage_question_attention": passage_question_attention, "span_start_logits": span_start_logits, "span_start_probs": span_start_probs, "span_end_logits": span_end_logits, "span_end_probs": span_end_probs, "best_span": best_span, } # Compute the loss for training. if span_start is not None: loss = nll_loss(util.masked_log_softmax(span_start_logits, passage_mask), span_start.squeeze(-1)) self._span_start_accuracy(span_start_logits, span_start.squeeze(-1)) loss += nll_loss(util.masked_log_softmax(span_end_logits, passage_mask), span_end.squeeze(-1)) self._span_end_accuracy(span_end_logits, span_end.squeeze(-1)) self._span_accuracy(best_span, torch.stack([span_start, span_end], -1)) output_dict["loss"] = loss # Compute the EM and F1 on SQuAD and add the tokenized input to the output. if metadata is not None: output_dict['best_span_str'] = [] question_tokens = [] passage_tokens = [] for i in range(batch_size): question_tokens.append(metadata[i]['question_tokens']) passage_tokens.append(metadata[i]['passage_tokens']) passage_str = metadata[i]['original_passage'] offsets = metadata[i]['token_offsets'] predicted_span = tuple(best_span[i].detach().cpu().numpy()) start_offset = offsets[predicted_span[0]][0] end_offset = offsets[predicted_span[1]][1] best_span_string = passage_str[start_offset:end_offset] output_dict['best_span_str'].append(best_span_string) answer_texts = metadata[i].get('answer_texts', []) if answer_texts: self._squad_metrics(best_span_string, answer_texts) output_dict['question_tokens'] = question_tokens output_dict['passage_tokens'] = passage_tokens return output_dict
def forward(self, # pylint: disable=arguments-differ inputs: torch.Tensor, mask: torch.LongTensor = None) -> torch.FloatTensor: """ Parameters ---------- inputs : ``torch.FloatTensor``, required. A tensor of shape (batch_size, timesteps, input_dim) mask : ``torch.FloatTensor``, optional (default = None). A tensor of shape (batch_size, timesteps). Returns ------- A tensor of shape (batch_size, timesteps, output_projection_dim), where output_projection_dim = input_dim by default. """ num_heads = self._num_heads batch_size, timesteps, _ = inputs.size() if mask is None: mask = inputs.new_ones(batch_size, timesteps) # Shape (batch_size, timesteps, 2 * attention_dim + values_dim) combined_projection = self._combined_projection(inputs) # split by attention dim - if values_dim > attention_dim, we will get more # than 3 elements returned. All of the rest are the values vector, so we # just concatenate them back together again below. queries, keys, *values = combined_projection.split(self._attention_dim, -1) queries = queries.contiguous() keys = keys.contiguous() values = torch.cat(values, -1).contiguous() # Shape (num_heads * batch_size, timesteps, values_dim / num_heads) values_per_head = values.view(batch_size, timesteps, num_heads, int(self._values_dim/num_heads)) values_per_head = values_per_head.transpose(1, 2).contiguous() values_per_head = values_per_head.view(batch_size * num_heads, timesteps, int(self._values_dim/num_heads)) # Shape (num_heads * batch_size, timesteps, attention_dim / num_heads) queries_per_head = queries.view(batch_size, timesteps, num_heads, int(self._attention_dim/num_heads)) queries_per_head = queries_per_head.transpose(1, 2).contiguous() queries_per_head = queries_per_head.view(batch_size * num_heads, timesteps, int(self._attention_dim/num_heads)) # Shape (num_heads * batch_size, timesteps, attention_dim / num_heads) keys_per_head = keys.view(batch_size, timesteps, num_heads, int(self._attention_dim/num_heads)) keys_per_head = keys_per_head.transpose(1, 2).contiguous() keys_per_head = keys_per_head.view(batch_size * num_heads, timesteps, int(self._attention_dim/num_heads)) # shape (num_heads * batch_size, timesteps, timesteps) scaled_similarities = torch.bmm(queries_per_head, keys_per_head.transpose(1, 2)) / self._scale # shape (num_heads * batch_size, timesteps, timesteps) # Normalise the distributions, using the same mask for all heads. attention = last_dim_softmax(scaled_similarities, mask.repeat(1, num_heads).view(batch_size * num_heads, timesteps)) attention = self._attention_dropout(attention) # Take a weighted sum of the values with respect to the attention # distributions for each element in the num_heads * batch_size dimension. # shape (num_heads * batch_size, timesteps, values_dim/num_heads) outputs = weighted_sum(values_per_head, attention) # Reshape back to original shape (batch_size, timesteps, values_dim) # shape (batch_size, num_heads, timesteps, values_dim/num_heads) outputs = outputs.view(batch_size, num_heads, timesteps, int(self._values_dim / num_heads)) # shape (batch_size, timesteps, num_heads, values_dim/num_heads) outputs = outputs.transpose(1, 2).contiguous() # shape (batch_size, timesteps, values_dim) outputs = outputs.view(batch_size, timesteps, self._values_dim) # Project back to original input size. # shape (batch_size, timesteps, input_size) outputs = self._output_projection(outputs) return outputs
def forward(self, # type: ignore question: Dict[str, torch.LongTensor], passage: Dict[str, torch.LongTensor], span_start: torch.IntTensor = None, span_end: torch.IntTensor = None, p1_answer_marker: torch.IntTensor = None, p2_answer_marker: torch.IntTensor = None, p3_answer_marker: torch.IntTensor = None, yesno_list: torch.IntTensor = None, followup_list: torch.IntTensor = None, metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ Parameters ---------- question : Dict[str, torch.LongTensor] From a ``TextField``. passage : Dict[str, torch.LongTensor] From a ``TextField``. The model assumes that this passage contains the answer to the question, and predicts the beginning and ending positions of the answer within the passage. span_start : ``torch.IntTensor``, optional From an ``IndexField``. This is one of the things we are trying to predict - the beginning position of the answer with the passage. This is an `inclusive` token index. If this is given, we will compute a loss that gets included in the output dictionary. span_end : ``torch.IntTensor``, optional From an ``IndexField``. This is one of the things we are trying to predict - the ending position of the answer with the passage. This is an `inclusive` token index. If this is given, we will compute a loss that gets included in the output dictionary. p1_answer_marker : ``torch.IntTensor``, optional This is one of the inputs, but only when num_context_answers > 0. This is a tensor that has a shape [batch_size, max_qa_count, max_passage_length]. Most passage token will have assigned 'O', except the passage tokens belongs to the previous answer in the dialog, which will be assigned labels such as <1_start>, <1_in>, <1_end>. For more details, look into dataset_readers/util/make_reading_comprehension_instance_quac p2_answer_marker : ``torch.IntTensor``, optional This is one of the inputs, but only when num_context_answers > 1. It is similar to p1_answer_marker, but marking previous previous answer in passage. p3_answer_marker : ``torch.IntTensor``, optional This is one of the inputs, but only when num_context_answers > 2. It is similar to p1_answer_marker, but marking previous previous previous answer in passage. yesno_list : ``torch.IntTensor``, optional This is one of the outputs that we are trying to predict. Three way classification (the yes/no/not a yes no question). followup_list : ``torch.IntTensor``, optional This is one of the outputs that we are trying to predict. Three way classification (followup / maybe followup / don't followup). metadata : ``List[Dict[str, Any]]``, optional If present, this should contain the question ID, original passage text, and token offsets into the passage for each instance in the batch. We use this for computing official metrics using the official SQuAD evaluation script. The length of this list should be the batch size, and each dictionary should have the keys ``id``, ``original_passage``, and ``token_offsets``. If you only want the best span string and don't care about official metrics, you can omit the ``id`` key. Returns ------- An output dictionary consisting of the followings. Each of the followings is a nested list because first iterates over dialog, then questions in dialog. qid : List[List[str]] A list of list, consisting of question ids. followup : List[List[int]] A list of list, consisting of continuation marker prediction index. (y :yes, m: maybe follow up, n: don't follow up) yesno : List[List[int]] A list of list, consisting of affirmation marker prediction index. (y :yes, x: not a yes/no question, n: np) best_span_str : List[List[str]] If sufficient metadata was provided for the instances in the batch, we also return the string from the original passage that the model thinks is the best answer to the question. loss : torch.FloatTensor, optional A scalar loss to be optimised. """ batch_size, max_qa_count, max_q_len, _ = question['token_characters'].size() total_qa_count = batch_size * max_qa_count qa_mask = torch.ge(followup_list, 0).view(total_qa_count) embedded_question = self._text_field_embedder(question, num_wrapping_dims=1) embedded_question = embedded_question.reshape(total_qa_count, max_q_len, self._text_field_embedder.get_output_dim()) embedded_question = self._variational_dropout(embedded_question) embedded_passage = self._variational_dropout(self._text_field_embedder(passage)) passage_length = embedded_passage.size(1) question_mask = util.get_text_field_mask(question, num_wrapping_dims=1).float() question_mask = question_mask.reshape(total_qa_count, max_q_len) passage_mask = util.get_text_field_mask(passage).float() repeated_passage_mask = passage_mask.unsqueeze(1).repeat(1, max_qa_count, 1) repeated_passage_mask = repeated_passage_mask.view(total_qa_count, passage_length) if self._num_context_answers > 0: # Encode question turn number inside the dialog into question embedding. question_num_ind = util.get_range_vector(max_qa_count, util.get_device_of(embedded_question)) question_num_ind = question_num_ind.unsqueeze(-1).repeat(1, max_q_len) question_num_ind = question_num_ind.unsqueeze(0).repeat(batch_size, 1, 1) question_num_ind = question_num_ind.reshape(total_qa_count, max_q_len) question_num_marker_emb = self._question_num_marker(question_num_ind) embedded_question = torch.cat([embedded_question, question_num_marker_emb], dim=-1) # Encode the previous answers in passage embedding. repeated_embedded_passage = embedded_passage.unsqueeze(1).repeat(1, max_qa_count, 1, 1). \ view(total_qa_count, passage_length, self._text_field_embedder.get_output_dim()) # batch_size * max_qa_count, passage_length, word_embed_dim p1_answer_marker = p1_answer_marker.view(total_qa_count, passage_length) p1_answer_marker_emb = self._prev_ans_marker(p1_answer_marker) repeated_embedded_passage = torch.cat([repeated_embedded_passage, p1_answer_marker_emb], dim=-1) if self._num_context_answers > 1: p2_answer_marker = p2_answer_marker.view(total_qa_count, passage_length) p2_answer_marker_emb = self._prev_ans_marker(p2_answer_marker) repeated_embedded_passage = torch.cat([repeated_embedded_passage, p2_answer_marker_emb], dim=-1) if self._num_context_answers > 2: p3_answer_marker = p3_answer_marker.view(total_qa_count, passage_length) p3_answer_marker_emb = self._prev_ans_marker(p3_answer_marker) repeated_embedded_passage = torch.cat([repeated_embedded_passage, p3_answer_marker_emb], dim=-1) repeated_encoded_passage = self._variational_dropout(self._phrase_layer(repeated_embedded_passage, repeated_passage_mask)) else: encoded_passage = self._variational_dropout(self._phrase_layer(embedded_passage, passage_mask)) repeated_encoded_passage = encoded_passage.unsqueeze(1).repeat(1, max_qa_count, 1, 1) repeated_encoded_passage = repeated_encoded_passage.view(total_qa_count, passage_length, self._encoding_dim) encoded_question = self._variational_dropout(self._phrase_layer(embedded_question, question_mask)) # Shape: (batch_size * max_qa_count, passage_length, question_length) passage_question_similarity = self._matrix_attention(repeated_encoded_passage, encoded_question) # Shape: (batch_size * max_qa_count, passage_length, question_length) passage_question_attention = util.masked_softmax(passage_question_similarity, question_mask) # Shape: (batch_size * max_qa_count, passage_length, encoding_dim) passage_question_vectors = util.weighted_sum(encoded_question, passage_question_attention) # We replace masked values with something really negative here, so they don't affect the # max below. masked_similarity = util.replace_masked_values(passage_question_similarity, question_mask.unsqueeze(1), -1e7) question_passage_similarity = masked_similarity.max(dim=-1)[0].squeeze(-1) question_passage_attention = util.masked_softmax(question_passage_similarity, repeated_passage_mask) # Shape: (batch_size * max_qa_count, encoding_dim) question_passage_vector = util.weighted_sum(repeated_encoded_passage, question_passage_attention) tiled_question_passage_vector = question_passage_vector.unsqueeze(1).expand(total_qa_count, passage_length, self._encoding_dim) # Shape: (batch_size * max_qa_count, passage_length, encoding_dim * 4) final_merged_passage = torch.cat([repeated_encoded_passage, passage_question_vectors, repeated_encoded_passage * passage_question_vectors, repeated_encoded_passage * tiled_question_passage_vector], dim=-1) final_merged_passage = F.relu(self._merge_atten(final_merged_passage)) residual_layer = self._variational_dropout(self._residual_encoder(final_merged_passage, repeated_passage_mask)) self_attention_matrix = self._self_attention(residual_layer, residual_layer) mask = repeated_passage_mask.reshape(total_qa_count, passage_length, 1) \ * repeated_passage_mask.reshape(total_qa_count, 1, passage_length) self_mask = torch.eye(passage_length, passage_length, device=self_attention_matrix.device) self_mask = self_mask.reshape(1, passage_length, passage_length) mask = mask * (1 - self_mask) self_attention_probs = util.masked_softmax(self_attention_matrix, mask) # (batch, passage_len, passage_len) * (batch, passage_len, dim) -> (batch, passage_len, dim) self_attention_vecs = torch.matmul(self_attention_probs, residual_layer) self_attention_vecs = torch.cat([self_attention_vecs, residual_layer, residual_layer * self_attention_vecs], dim=-1) residual_layer = F.relu(self._merge_self_attention(self_attention_vecs)) final_merged_passage = final_merged_passage + residual_layer # batch_size * maxqa_pair_len * max_passage_len * 200 final_merged_passage = self._variational_dropout(final_merged_passage) start_rep = self._span_start_encoder(final_merged_passage, repeated_passage_mask) span_start_logits = self._span_start_predictor(start_rep).squeeze(-1) end_rep = self._span_end_encoder(torch.cat([final_merged_passage, start_rep], dim=-1), repeated_passage_mask) span_end_logits = self._span_end_predictor(end_rep).squeeze(-1) span_yesno_logits = self._span_yesno_predictor(end_rep).squeeze(-1) span_followup_logits = self._span_followup_predictor(end_rep).squeeze(-1) span_start_logits = util.replace_masked_values(span_start_logits, repeated_passage_mask, -1e7) # batch_size * maxqa_len_pair, max_document_len span_end_logits = util.replace_masked_values(span_end_logits, repeated_passage_mask, -1e7) best_span = self._get_best_span_yesno_followup(span_start_logits, span_end_logits, span_yesno_logits, span_followup_logits, self._max_span_length) output_dict: Dict[str, Any] = {} # Compute the loss. if span_start is not None: loss = nll_loss(util.masked_log_softmax(span_start_logits, repeated_passage_mask), span_start.view(-1), ignore_index=-1) self._span_start_accuracy(span_start_logits, span_start.view(-1), mask=qa_mask) loss += nll_loss(util.masked_log_softmax(span_end_logits, repeated_passage_mask), span_end.view(-1), ignore_index=-1) self._span_end_accuracy(span_end_logits, span_end.view(-1), mask=qa_mask) self._span_accuracy(best_span[:, 0:2], torch.stack([span_start, span_end], -1).view(total_qa_count, 2), mask=qa_mask.unsqueeze(1).expand(-1, 2).long()) # add a select for the right span to compute loss gold_span_end_loc = [] span_end = span_end.view(total_qa_count).squeeze().data.cpu().numpy() for i in range(0, total_qa_count): gold_span_end_loc.append(max(span_end[i] * 3 + i * passage_length * 3, 0)) gold_span_end_loc.append(max(span_end[i] * 3 + i * passage_length * 3 + 1, 0)) gold_span_end_loc.append(max(span_end[i] * 3 + i * passage_length * 3 + 2, 0)) gold_span_end_loc = span_start.new(gold_span_end_loc) pred_span_end_loc = [] for i in range(0, total_qa_count): pred_span_end_loc.append(max(best_span[i][1] * 3 + i * passage_length * 3, 0)) pred_span_end_loc.append(max(best_span[i][1] * 3 + i * passage_length * 3 + 1, 0)) pred_span_end_loc.append(max(best_span[i][1] * 3 + i * passage_length * 3 + 2, 0)) predicted_end = span_start.new(pred_span_end_loc) _yesno = span_yesno_logits.view(-1).index_select(0, gold_span_end_loc).view(-1, 3) _followup = span_followup_logits.view(-1).index_select(0, gold_span_end_loc).view(-1, 3) loss += nll_loss(F.log_softmax(_yesno, dim=-1), yesno_list.view(-1), ignore_index=-1) loss += nll_loss(F.log_softmax(_followup, dim=-1), followup_list.view(-1), ignore_index=-1) _yesno = span_yesno_logits.view(-1).index_select(0, predicted_end).view(-1, 3) _followup = span_followup_logits.view(-1).index_select(0, predicted_end).view(-1, 3) self._span_yesno_accuracy(_yesno, yesno_list.view(-1), mask=qa_mask) self._span_followup_accuracy(_followup, followup_list.view(-1), mask=qa_mask) output_dict["loss"] = loss # Compute F1 and preparing the output dictionary. output_dict['best_span_str'] = [] output_dict['qid'] = [] output_dict['followup'] = [] output_dict['yesno'] = [] best_span_cpu = best_span.detach().cpu().numpy() for i in range(batch_size): passage_str = metadata[i]['original_passage'] offsets = metadata[i]['token_offsets'] f1_score = 0.0 per_dialog_best_span_list = [] per_dialog_yesno_list = [] per_dialog_followup_list = [] per_dialog_query_id_list = [] for per_dialog_query_index, (iid, answer_texts) in enumerate( zip(metadata[i]["instance_id"], metadata[i]["answer_texts_list"])): predicted_span = tuple(best_span_cpu[i * max_qa_count + per_dialog_query_index]) start_offset = offsets[predicted_span[0]][0] end_offset = offsets[predicted_span[1]][1] yesno_pred = predicted_span[2] followup_pred = predicted_span[3] per_dialog_yesno_list.append(yesno_pred) per_dialog_followup_list.append(followup_pred) per_dialog_query_id_list.append(iid) best_span_string = passage_str[start_offset:end_offset] per_dialog_best_span_list.append(best_span_string) if answer_texts: if len(answer_texts) > 1: t_f1 = [] # Compute F1 over N-1 human references and averages the scores. for answer_index in range(len(answer_texts)): idxes = list(range(len(answer_texts))) idxes.pop(answer_index) refs = [answer_texts[z] for z in idxes] t_f1.append(squad_eval.metric_max_over_ground_truths(squad_eval.f1_score, best_span_string, refs)) f1_score = 1.0 * sum(t_f1) / len(t_f1) else: f1_score = squad_eval.metric_max_over_ground_truths(squad_eval.f1_score, best_span_string, answer_texts) self._official_f1(100 * f1_score) output_dict['qid'].append(per_dialog_query_id_list) output_dict['best_span_str'].append(per_dialog_best_span_list) output_dict['yesno'].append(per_dialog_yesno_list) output_dict['followup'].append(per_dialog_followup_list) return output_dict
def forward(self, s1, s2): # pylint: disable=arguments-differ """ Parameters ---------- s1 : Dict[str, torch.LongTensor] From a ``TextField``. s2 : Dict[str, torch.LongTensor] From a ``TextField``. The model assumes that this s2 contains the answer to the s1, and predicts the beginning and ending positions of the answer within the s2. Returns ------- pair_rep : torch.FloatTensor? Tensor representing the final output of the BiDAF model to be plugged into the next module """ s1_embs = self._highway_layer(self._text_field_embedder(s1)) s2_embs = self._highway_layer(self._text_field_embedder(s2)) if self._elmo is not None: s1_elmo_embs = self._elmo(s1['elmo']) s2_elmo_embs = self._elmo(s2['elmo']) if "words" in s1: s1_embs = torch.cat([s1_embs, s1_elmo_embs['elmo_representations'][0]], dim=-1) s2_embs = torch.cat([s2_embs, s2_elmo_embs['elmo_representations'][0]], dim=-1) else: s1_embs = s1_elmo_embs['elmo_representations'][0] s2_embs = s2_elmo_embs['elmo_representations'][0] if self._cove is not None: s1_lens = torch.ne(s1['words'], self.pad_idx).long().sum(dim=-1).data s2_lens = torch.ne(s2['words'], self.pad_idx).long().sum(dim=-1).data s1_cove_embs = self._cove(s1['words'], s1_lens) s1_embs = torch.cat([s1_embs, s1_cove_embs], dim=-1) s2_cove_embs = self._cove(s2['words'], s2_lens) s2_embs = torch.cat([s2_embs, s2_cove_embs], dim=-1) s1_embs = self._dropout(s1_embs) s2_embs = self._dropout(s2_embs) if self._mask_lstms: s1_mask = s1_lstm_mask = util.get_text_field_mask(s1).float() s2_mask = s2_lstm_mask = util.get_text_field_mask(s2).float() s1_mask_2 = util.get_text_field_mask(s1).float() s2_mask_2 = util.get_text_field_mask(s2).float() else: s1_lstm_mask, s2_lstm_mask, s2_lstm_mask_2 = None, None, None s1_enc = self._phrase_layer(s1_embs, s1_lstm_mask) s2_enc = self._phrase_layer(s2_embs, s2_lstm_mask) # Similarity matrix # Shape: (batch_size, s2_length, s1_length) similarity_mat = self._matrix_attention(s2_enc, s1_enc) # s2 representation # Shape: (batch_size, s2_length, s1_length) s2_s1_attention = util.last_dim_softmax(similarity_mat, s1_mask) # Shape: (batch_size, s2_length, encoding_dim) s2_s1_vectors = util.weighted_sum(s1_enc, s2_s1_attention) # batch_size, seq_len, 4*enc_dim s2_w_context = torch.cat([s2_enc, s2_s1_vectors], 2) # s1 representation, using same attn method as for the s2 representation s1_s2_attention = util.last_dim_softmax(similarity_mat.transpose(1, 2).contiguous(), s2_mask) # Shape: (batch_size, s1_length, encoding_dim) s1_s2_vectors = util.weighted_sum(s2_enc, s1_s2_attention) s1_w_context = torch.cat([s1_enc, s1_s2_vectors], 2) if self._elmo is not None and self._deep_elmo: s1_w_context = torch.cat([s1_w_context, s1_elmo_embs['elmo_representations'][1]], dim=-1) s2_w_context = torch.cat([s2_w_context, s2_elmo_embs['elmo_representations'][1]], dim=-1) s1_w_context = self._dropout(s1_w_context) s2_w_context = self._dropout(s2_w_context) modeled_s2 = self._dropout(self._modeling_layer(s2_w_context, s2_lstm_mask)) s2_mask_2 = s2_mask_2.unsqueeze(dim=-1) modeled_s2.data.masked_fill_(1 - s2_mask_2.byte().data, -float('inf')) s2_enc_attn = modeled_s2.max(dim=1)[0] modeled_s1 = self._dropout(self._modeling_layer(s1_w_context, s1_lstm_mask)) s1_mask_2 = s1_mask_2.unsqueeze(dim=-1) modeled_s1.data.masked_fill_(1 - s1_mask_2.byte().data, -float('inf')) s1_enc_attn = modeled_s1.max(dim=1)[0] return torch.cat([s1_enc_attn, s2_enc_attn, torch.abs(s1_enc_attn - s2_enc_attn), s1_enc_attn * s2_enc_attn], 1)