def __init__(self, device, inp_dim, hidden_dim, nenc_lay, dropout): super().__init__() self.hidden_dim = hidden_dim self.enc_blstm = PytorchSeq2SeqWrapper( torch.nn.LSTM(inp_dim, hidden_dim, batch_first=True, bidirectional=True, num_layers=nenc_lay)) self._span_encoder = BidirectionalEndpointSpanExtractor( self.enc_blstm.get_output_dim()) self._dropout = torch.nn.Dropout(p=dropout) self.device = device
def test_forward_raises_with_invalid_indices(self): sequence_tensor = torch.randn([2, 5, 8]) extractor = BidirectionalEndpointSpanExtractor(input_dim=8) indices = torch.LongTensor([[[-1, 3], [7, 4]], [[0, 12], [0, -1]]]) with pytest.raises(ValueError): _ = extractor(sequence_tensor, indices)
class EncWord2Sent(torch.nn.Module): # Normal BLSTM encoder from word embedding to hidden rep def __init__(self, device, inp_dim, hidden_dim, nenc_lay, dropout): super().__init__() self.hidden_dim = hidden_dim self.enc_blstm = PytorchSeq2SeqWrapper( torch.nn.LSTM(inp_dim, hidden_dim, batch_first=True, bidirectional=True, num_layers=nenc_lay)) self._span_encoder = BidirectionalEndpointSpanExtractor( self.enc_blstm.get_output_dim()) self._dropout = torch.nn.Dropout(p=dropout) self.device = device def get_output_dim(self): return self.enc_blstm.get_output_dim() def forward(self, context, context_msk): """ :param context: [batch, t, inp_dim] :param context_msk: [batch, t] [0,1] :return: blstm_output [batch, t, hid_dim*2] avg_blstm_out [batch, hid*2] """ batch_size = context.size()[0] blstm_output = self._dropout(self.enc_blstm(context, context_msk)) # blstm context_msk[:, 0] = 1 # context_msk = [batch, t] context_mask_sum = torch.sum(context_msk, dim=1) - 1 # context_mask_sum = [batch] [ 40, 4, 9, 0(sent len=1) , -1(sent len=0), -1] sum_of_mask = context_mask_sum.unsqueeze(dim=1).unsqueeze(dim=2).long() span_idx = torch.ones( (batch_size), device=self.device, dtype=torch.long) * -1 # [ [-1] [-1] [-1] [-1] ] # according to sum_of_mask valid_bit = (context_mask_sum >= 0).long() span_idx = span_idx + valid_bit span_idx = span_idx.view((batch_size, 1, 1)) span_idx = torch.cat([span_idx, sum_of_mask], dim=2).long() # Span module: (batch_size, sequence_length, embedding_size) # (batch_size, num_spans, 2) attended_text_embeddings = self._span_encoder.forward( sequence_tensor=blstm_output, span_indices=span_idx, sequence_mask=context_msk, span_indices_mask=valid_bit.unsqueeze(1)) # attended_text_embeddings: batch, 1, dim attended_text_embeddings = attended_text_embeddings.squeeze(1) # valid_len = context_msk.sum(dim=1).unsqueeze(1) # batchsz,1. # context_msk = context_msk.unsqueeze(2) # msked_blstm_out = context_msk * blstm_output attended_text_embeddings = self._dropout(attended_text_embeddings) return blstm_output, attended_text_embeddings
def test_forward_doesnt_raise_with_empty_sequence(self): # size: (batch_size=1, sequence_length=2, emb_dim=2) sequence_tensor = torch.FloatTensor([[[0., 0.], [0., 0.]]]) # size: (batch_size=1, sequence_length=2) sequence_mask = torch.LongTensor([[0, 0]]) # size: (batch_size=1, spans_count=1, 2) span_indices = torch.LongTensor([[[-1, -1]]]) # size: (batch_size=1, spans_count=1) span_indices_mask = torch.LongTensor([[0]]) extractor = BidirectionalEndpointSpanExtractor( input_dim=2, forward_combination="x,y", backward_combination="x,y") span_representations = extractor(sequence_tensor, span_indices, sequence_mask=sequence_mask, span_indices_mask=span_indices_mask) numpy.testing.assert_array_equal( span_representations.detach(), torch.FloatTensor([[[0., 0., 0., 0.]]]))
def test_correct_sequence_elements_are_embedded(self): sequence_tensor = torch.randn([2, 5, 8]) # concatentate start and end points together to form our representation # for both the forward and backward directions. extractor = BidirectionalEndpointSpanExtractor( input_dim=8, forward_combination="x,y", backward_combination="x,y") indices = torch.LongTensor([[[1, 3], [2, 4]], [[0, 2], [3, 4]]]) span_representations = extractor(sequence_tensor, indices) assert list(span_representations.size()) == [2, 2, 16] assert extractor.get_output_dim() == 16 assert extractor.get_input_dim() == 8 # We just concatenated the start and end embeddings together, so # we can check they match the original indices if we split them apart. ( forward_start_embeddings, forward_end_embeddings, backward_start_embeddings, backward_end_embeddings, ) = span_representations.split(4, -1) forward_sequence_tensor, backward_sequence_tensor = sequence_tensor.split( 4, -1) # Forward direction => subtract 1 from start indices to make them exlusive. correct_forward_start_indices = torch.LongTensor([[0, 1], [-1, 2]]) # This index should be -1, so it will be replaced with a sentinel. Here, # we'll set it to a value other than -1 so we can index select the indices and # replace it later. correct_forward_start_indices[1, 0] = 1 # Forward direction => end indices are the same. correct_forward_end_indices = torch.LongTensor([[3, 4], [2, 4]]) # Backward direction => start indices are exclusive, so add 1 to the end indices. correct_backward_start_indices = torch.LongTensor([[4, 5], [3, 5]]) # These exclusive end indices are outside the tensor, so will be replaced with the end sentinel. # Here we replace them with ones so we can index select using these indices without torch # complaining. correct_backward_start_indices[0, 1] = 1 correct_backward_start_indices[1, 1] = 1 # Backward direction => end indices are inclusive and equal to the forward start indices. correct_backward_end_indices = torch.LongTensor([[1, 2], [0, 3]]) correct_forward_start_embeddings = batched_index_select( forward_sequence_tensor.contiguous(), correct_forward_start_indices) # This element had sequence_tensor index of 0, so it's exclusive index is the start sentinel. correct_forward_start_embeddings[1, 0] = extractor._start_sentinel.data numpy.testing.assert_array_equal( forward_start_embeddings.data.numpy(), correct_forward_start_embeddings.data.numpy()) correct_forward_end_embeddings = batched_index_select( forward_sequence_tensor.contiguous(), correct_forward_end_indices) numpy.testing.assert_array_equal( forward_end_embeddings.data.numpy(), correct_forward_end_embeddings.data.numpy()) correct_backward_end_embeddings = batched_index_select( backward_sequence_tensor.contiguous(), correct_backward_end_indices) numpy.testing.assert_array_equal( backward_end_embeddings.data.numpy(), correct_backward_end_embeddings.data.numpy()) correct_backward_start_embeddings = batched_index_select( backward_sequence_tensor.contiguous(), correct_backward_start_indices) # This element had sequence_tensor index == sequence_tensor.size(1), # so it's exclusive index is the end sentinel. correct_backward_start_embeddings[0, 1] = extractor._end_sentinel.data correct_backward_start_embeddings[1, 1] = extractor._end_sentinel.data numpy.testing.assert_array_equal( backward_start_embeddings.data.numpy(), correct_backward_start_embeddings.data.numpy())
def test_raises_on_odd_input_dimension(self): with pytest.raises(ConfigurationError): _ = BidirectionalEndpointSpanExtractor(7)
def test_correct_sequence_elements_are_embedded_with_a_masked_sequence( self): sequence_tensor = torch.randn([2, 5, 8]) # concatentate start and end points together to form our representation # for both the forward and backward directions. extractor = BidirectionalEndpointSpanExtractor( input_dim=8, forward_combination="x,y", backward_combination="x,y") indices = torch.LongTensor([ [[1, 3], [2, 4]], # This span has an end index at the # end of the padded sequence. [[0, 2], [0, 1]], ]) sequence_mask = torch.LongTensor([[1, 1, 1, 1, 1], [1, 1, 1, 0, 0]]) span_representations = extractor(sequence_tensor, indices, sequence_mask=sequence_mask) # We just concatenated the start and end embeddings together, so # we can check they match the original indices if we split them apart. ( forward_start_embeddings, forward_end_embeddings, backward_start_embeddings, backward_end_embeddings, ) = span_representations.split(4, -1) forward_sequence_tensor, backward_sequence_tensor = sequence_tensor.split( 4, -1) # Forward direction => subtract 1 from start indices to make them exlusive. correct_forward_start_indices = torch.LongTensor([[0, 1], [-1, -1]]) # These indices should be -1, so they'll be replaced with a sentinel. Here, # we'll set them to a value other than -1 so we can index select the indices and # replace them later. correct_forward_start_indices[1, 0] = 1 correct_forward_start_indices[1, 1] = 1 # Forward direction => end indices are the same. correct_forward_end_indices = torch.LongTensor([[3, 4], [2, 1]]) # Backward direction => start indices are exclusive, so add 1 to the end indices. correct_backward_start_indices = torch.LongTensor([[4, 5], [3, 2]]) # These exclusive backward start indices are outside the tensor, so will be replaced # with the end sentinel. Here we replace them with ones so we can index select using # these indices without torch complaining. correct_backward_start_indices[0, 1] = 1 # Backward direction => end indices are inclusive and equal to the forward start indices. correct_backward_end_indices = torch.LongTensor([[1, 2], [0, 0]]) correct_forward_start_embeddings = batched_index_select( forward_sequence_tensor.contiguous(), correct_forward_start_indices) # This element had sequence_tensor index of 0, so it's exclusive index is the start sentinel. correct_forward_start_embeddings[1, 0] = extractor._start_sentinel.data correct_forward_start_embeddings[1, 1] = extractor._start_sentinel.data numpy.testing.assert_array_equal( forward_start_embeddings.data.numpy(), correct_forward_start_embeddings.data.numpy()) correct_forward_end_embeddings = batched_index_select( forward_sequence_tensor.contiguous(), correct_forward_end_indices) numpy.testing.assert_array_equal( forward_end_embeddings.data.numpy(), correct_forward_end_embeddings.data.numpy()) correct_backward_end_embeddings = batched_index_select( backward_sequence_tensor.contiguous(), correct_backward_end_indices) numpy.testing.assert_array_equal( backward_end_embeddings.data.numpy(), correct_backward_end_embeddings.data.numpy()) correct_backward_start_embeddings = batched_index_select( backward_sequence_tensor.contiguous(), correct_backward_start_indices) # This element had sequence_tensor index == sequence_tensor.size(1), # so it's exclusive index is the end sentinel. correct_backward_start_embeddings[0, 1] = extractor._end_sentinel.data # This element has sequence_tensor index == the masked length of the batch element, # so it should be the end_sentinel even though it isn't greater than sequence_tensor.size(1). correct_backward_start_embeddings[1, 0] = extractor._end_sentinel.data numpy.testing.assert_array_equal( backward_start_embeddings.data.numpy(), correct_backward_start_embeddings.data.numpy())
def test_correct_sequence_elements_are_embedded(self): sequence_tensor = Variable(torch.randn([2, 5, 8])) # concatentate start and end points together to form our representation # for both the forward and backward directions. extractor = BidirectionalEndpointSpanExtractor(input_dim=8, forward_combination="x,y", backward_combination="x,y") indices = Variable(torch.LongTensor([[[1, 3], [2, 4]], [[0, 2], [3, 4]]])) span_representations = extractor(sequence_tensor, indices) assert list(span_representations.size()) == [2, 2, 16] assert extractor.get_output_dim() == 16 assert extractor.get_input_dim() == 8 # We just concatenated the start and end embeddings together, so # we can check they match the original indices if we split them apart. (forward_start_embeddings, forward_end_embeddings, backward_start_embeddings, backward_end_embeddings) = span_representations.split(4, -1) forward_sequence_tensor, backward_sequence_tensor = sequence_tensor.split(4, -1) # Forward direction => subtract 1 from start indices to make them exlusive. correct_forward_start_indices = Variable(torch.LongTensor([[0, 1], [-1, 2]])) # This index should be -1, so it will be replaced with a sentinel. Here, # we'll set it to a value other than -1 so we can index select the indices and # replace it later. correct_forward_start_indices[1, 0] = 1 # Forward direction => end indices are the same. correct_forward_end_indices = Variable(torch.LongTensor([[3, 4], [2, 4]])) # Backward direction => start indices are exclusive, so add 1 to the end indices. correct_backward_start_indices = Variable(torch.LongTensor([[4, 5], [3, 5]])) # These exclusive end indices are outside the tensor, so will be replaced with the end sentinel. # Here we replace them with ones so we can index select using these indices without torch # complaining. correct_backward_start_indices[0, 1] = 1 correct_backward_start_indices[1, 1] = 1 # Backward direction => end indices are inclusive and equal to the forward start indices. correct_backward_end_indices = Variable(torch.LongTensor([[1, 2], [0, 3]])) correct_forward_start_embeddings = batched_index_select(forward_sequence_tensor.contiguous(), correct_forward_start_indices) # This element had sequence_tensor index of 0, so it's exclusive index is the start sentinel. correct_forward_start_embeddings[1, 0] = extractor._start_sentinel.data numpy.testing.assert_array_equal(forward_start_embeddings.data.numpy(), correct_forward_start_embeddings.data.numpy()) correct_forward_end_embeddings = batched_index_select(forward_sequence_tensor.contiguous(), correct_forward_end_indices) numpy.testing.assert_array_equal(forward_end_embeddings.data.numpy(), correct_forward_end_embeddings.data.numpy()) correct_backward_end_embeddings = batched_index_select(backward_sequence_tensor.contiguous(), correct_backward_end_indices) numpy.testing.assert_array_equal(backward_end_embeddings.data.numpy(), correct_backward_end_embeddings.data.numpy()) correct_backward_start_embeddings = batched_index_select(backward_sequence_tensor.contiguous(), correct_backward_start_indices) # This element had sequence_tensor index == sequence_tensor.size(1), # so it's exclusive index is the end sentinel. correct_backward_start_embeddings[0, 1] = extractor._end_sentinel.data correct_backward_start_embeddings[1, 1] = extractor._end_sentinel.data numpy.testing.assert_array_equal(backward_start_embeddings.data.numpy(), correct_backward_start_embeddings.data.numpy())
"lang": { "inputs": "lang" } }, allow_unmatched_keys=True) encoder: Seq2VecEncoder = PytorchSeq2SeqWrapper( nn.LSTM(word_embedder.get_output_dim(), encoder_output_dim, 2, dropout=0.4, bidirectional=True, batch_first=True)) # span_extractor: SpanExtractor = SelfAttentiveSpanExtractor(input_dim=encoder.get_output_dim()) span_extractor: SpanExtractor = BidirectionalEndpointSpanExtractor( input_dim=encoder.get_output_dim()) # probably the best solution is to make it like a factory, to add a get_decoder function that get as input # vocab and needed dimnsion. span_decoder = Topdown_Span_Parser_Factory() remote_parser = Basic_Remote_Parser_Factory(remote_parser_mlp_dim) model = UccaSpanParser(word_embedder, encoder, span_decoder, span_extractor, remote_parser, UccaScores(), vocab) if torch.cuda.is_available(): cuda_device = list(range(torch.cuda.device_count())) model = model.cuda(cuda_device[0]) else: cuda_device = -1 trainer = Trainer(model=model,