def test_mask_is_computed_correctly(self):
background_input = Input(shape=(None, 3), dtype='int32')
embedding = TimeDistributedEmbedding(input_dim=3,
output_dim=2,
mask_zero=True)
embedded_background = embedding(background_input)
encoded_background = BOWEncoder(units=2)(embedded_background)
encoded_background_with_mask = AddEncoderMask()(
[encoded_background, embedded_background])
mask_output = OutputMask()(encoded_background_with_mask)
model = DeepQaModel(inputs=[background_input], outputs=mask_output)
test_background = numpy.asarray([[
[0, 0, 0],
[2, 2, 2],
[0, 0, 0],
[0, 1, 2],
[1, 0, 0],
[0, 0, 0],
[0, 1, 0],
[1, 1, 1],
]])
expected_mask = numpy.asarray([[0, 1, 0, 1, 1, 0, 1, 1]])
actual_mask = model.predict([test_background])
numpy.testing.assert_array_equal(expected_mask, actual_mask)
def test_on_masked_input(self):
# TODO(matt): I don't really like having to build the whole model up to the attention
# component here, but I'm not sure how to just test the selector with the right mask
# without going through this.
sentence_input = Input(shape=(3, ), dtype='int32')
background_input = Input(shape=(3, 3), dtype='int32')
embedding = TimeDistributedEmbedding(input_dim=3,
output_dim=2,
mask_zero=True)
embedded_sentence = embedding(sentence_input)
embedded_background = embedding(background_input)
encoder = BOWEncoder(output_dim=2)
encoded_sentence = encoder(embedded_sentence)
encoded_background = EncoderWrapper(encoder)(embedded_background)
merge_mode = lambda layer_outs: K.concatenate([
K.expand_dims(layer_outs[0], dim=1),
K.expand_dims(layer_outs[0], dim=1), layer_outs[1]
],
axis=1)
merge_masks = lambda mask_outs: K.concatenate([
K.expand_dims(K.zeros_like(mask_outs[1][:, 0]), dim=1),
K.expand_dims(K.zeros_like(mask_outs[1][:, 0]), dim=1), mask_outs[1
]
],
axis=1)
merged = merge([encoded_sentence, encoded_background],
mode=merge_mode,
output_shape=(5, 2),
output_mask=merge_masks)
merged_mask = OutputMask()(merged)
selector = DotProductKnowledgeSelector()
attention_weights = selector(merged)
model = DeepQaModel(input=[sentence_input, background_input],
output=[merged_mask, attention_weights])
model.summary(show_masks=True)
test_input = numpy.asarray([[2, 2, 2]])
test_background = numpy.asarray([[
[2, 2, 2],
[2, 2, 2],
[0, 0, 0],
]])
expected_mask = numpy.asarray([[0, 0, 1, 1, 0]])
expected_attention = numpy.asarray([[0.5, 0.5, 0.0]])
actual_mask, actual_attention = model.predict(
[test_input, test_background])
numpy.testing.assert_array_almost_equal(expected_mask, actual_mask)
numpy.testing.assert_array_almost_equal(expected_attention,
actual_attention)
def test_handles_multiple_masks(self):
# We'll use the SlotSimilarityTupleMatcher to test this, because it takes two masked
# inputs. Here we're using an input of shape (batch_size, num_options, num_tuples,
# num_slots, num_words).
tuple_input = Input(shape=(2, 3, 4, 5), dtype='int32')
tuple_input_2 = Input(shape=(2, 3, 4, 5), dtype='int32')
embedding = TimeDistributedEmbedding(input_dim=3,
output_dim=6,
mask_zero=True)
# shape is now (batch_size, num_options, num_tuples, num_slots, num_words, embedding_dim)
embedded_tuple = embedding(tuple_input)
embedded_tuple_2 = embedding(tuple_input_2)
encoder = EncoderWrapper(EncoderWrapper(EncoderWrapper(BOWEncoder())))
# shape is now (batch_size, num_options, num_tuples, num_slots, embedding_dim)
encoded_tuple = encoder(embedded_tuple)
encoded_tuple_2 = encoder(embedded_tuple_2)
# Shape of input to the tuple matcher is [(batch size, 2, 3, 4, 6), (batch size, 2, 3, 4, 6)]
# Shape of input_mask to the tuple matcher is [(batch size, 2, 3, 4), (batch size, 2, 3, 4)]
# Expected output mask shape (batch_size, 2, 3)
time_distributed = TimeDistributedWithMask(
TimeDistributedWithMask(
SlotSimilarityTupleMatcher({"type": "cosine_similarity"})))
time_distributed_output = time_distributed(
[encoded_tuple, encoded_tuple_2])
mask_output = OutputMask()(time_distributed_output)
model = DeepQaModel(input=[tuple_input, tuple_input_2],
output=mask_output)
zeros = [0, 0, 0, 0, 0]
non_zeros = [1, 1, 1, 1, 1]
# shape: (batch size, num_options, num_tuples, num_slots, num_words), or (1, 2, 3, 4, 5)
tuples1 = numpy.asarray([[[[zeros, zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[non_zeros, non_zeros, zeros, zeros]],
[[non_zeros, non_zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[zeros, zeros, zeros, zeros]]]])
tuples2 = numpy.asarray([[[[non_zeros, zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[zeros, zeros, zeros, zeros]],
[[non_zeros, non_zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros]]]])
actual_mask = model.predict([tuples1, tuples2])
expected_mask = numpy.asarray(
[[[0, 1, 0], [1, 1,
0]]]) # shape: (batch size, num_options, num_tuples)
assert actual_mask.shape == (1, 2, 3)
numpy.testing.assert_array_almost_equal(expected_mask, actual_mask)
def test_mask_is_computed_correctly(self):
background_input = Input(shape=(3, 3), dtype='int32')
embedding = Embedding(input_dim=3, output_dim=2, mask_zero=True)
embedded_background = embedding(background_input)
encoded_background = EncoderWrapper(BOWEncoder(units=2))(embedded_background)
mask_output = OutputMask()(encoded_background)
model = DeepQaModel(inputs=[background_input], outputs=mask_output)
test_background = numpy.asarray([
[
[0, 0, 0],
[2, 2, 2],
[0, 0, 0],
]
])
expected_mask = numpy.asarray([[0, 1, 0]])
actual_mask = model.predict([test_background])
numpy.testing.assert_array_almost_equal(expected_mask, actual_mask)
def test_returns_masks_if_no_input_mask(self):
# For this test, we use WordOverlapTupleMatcher, which takes no input mask, but
# returns an output mask. We're using an input of shape (batch_size, num_options,
# num_tuples, num_slots, num_words).
tuple_input = Input(shape=(2, 3, 4, 5), dtype='int32')
tuple_input_2 = Input(shape=(2, 3, 4, 5), dtype='int32')
# shape is (batch_size, num_options, num_tuples, num_slots, num_words)
# Shape of input to the tuple matcher is [(batch size, 2, 3, 4, 5), (batch size, 2, 3, 4, 5)]
# Shape of input_mask to the tuple matcher is [None, None]
# Expected output mask shape (batch_size, 2, 3)
time_distributed = TimeDistributedWithMask(
TimeDistributedWithMask(WordOverlapTupleMatcher()))
time_distributed_output = time_distributed(
[tuple_input, tuple_input_2])
mask_output = OutputMask()(time_distributed_output)
model = DeepQaModel(inputs=[tuple_input, tuple_input_2],
outputs=mask_output)
zeros = [0, 0, 0, 0, 0]
non_zeros = [1, 1, 1, 1, 1]
# shape: (batch size, num_options, num_tuples, num_slots, num_words), or (1, 2, 3, 4, 5)
tuples1 = numpy.asarray([[[[zeros, zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[non_zeros, non_zeros, zeros, zeros]],
[[non_zeros, non_zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[zeros, zeros, zeros, zeros]]]])
tuples2 = numpy.asarray([[[[non_zeros, zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[zeros, zeros, zeros, zeros]],
[[non_zeros, non_zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros]]]])
actual_mask = model.predict([tuples1, tuples2])
expected_mask = numpy.asarray(
[[[0, 1, 0], [1, 1,
0]]]) # shape: (batch size, num_options, num_tuples)
assert actual_mask.shape == (1, 2, 3)
numpy.testing.assert_array_almost_equal(expected_mask, actual_mask)
def _build_model(self):
# Input: (sentence, verb-span, entity-span)
# Output: (state-change-type, argument-tags)
# shape: (batch_size, text_length)
sentence_input = Input(shape=self._get_sentence_shape(),
dtype='int32',
name='word_array_input')
verb_input = Input(shape=self._get_sentence_shape(),
dtype='int32',
name='verb_array_input')
entity_input = Input(shape=self._get_sentence_shape(),
dtype='int32',
name='entity_array_input')
# shape: (batch_size, text_length, embedding_dim)
sentence_embedding = self._embed_input(sentence_input)
multiply_layer = Multiply()
verb_indices = multiply_layer([sentence_input, verb_input])
verb_embedding = self._embed_input(verb_indices)
entity_indices = multiply_layer([sentence_input, entity_input])
entity_embedding = self._embed_input(entity_indices)
# For state-change-type prediction: We first convert verb and entity into bag-of-words(BOW) representation
# and then apply a dense layer with soft-max activation to predict state change type.
bow_features = BOWEncoder()
average_layer = Average()
verb_entity_vector = average_layer(
[bow_features(verb_embedding),
bow_features(entity_embedding)])
state_change_type = Dense(
self.data_indexer.get_vocab_size("state_changes") - 2,
activation='softmax')(verb_entity_vector)
# For argument-tags prediction: We first convert a sentence to a sequence of word embeddings
# and then apply a stack of seq2seq encoders to finally predict a sequence of argument tags.
for i in range(self.num_stacked_rnns):
encoder = self._get_seq2seq_encoder(
name="encoder_{}".format(i),
fallback_behavior="use default params")
# shape still (batch_size, text_length, embedding_dim)
print("Inside _build_model:", sentence_embedding)
sentence_embedding = encoder(sentence_embedding)
# The -2 below is because we are ignoring the padding and unknown tokens that the
# DataIndexer has by default.
argument_tags = TimeDistributed(
Dense(self.data_indexer.get_vocab_size('tags') - 2,
activation='softmax'))(sentence_embedding)
return DeepQaModel(input=[sentence_input, verb_input, entity_input],
output=[state_change_type, argument_tags])
def test_mask_is_computed_correctly(self):
# TODO(matt): I don't really like having to build a model to test this, but I'm not sure of
# how else to do it.
background_input = Input(shape=(3, 3), dtype='int32')
embedding = TimeDistributedEmbedding(input_dim=3,
output_dim=2,
mask_zero=True)
embedded_background = embedding(background_input)
encoded_background = EncoderWrapper(
BOWEncoder(output_dim=2))(embedded_background)
mask_output = OutputMask()(encoded_background)
model = DeepQaModel(input=[background_input], output=mask_output)
test_background = numpy.asarray([[
[0, 0, 0],
[2, 2, 2],
[0, 0, 0],
]])
expected_mask = numpy.asarray([[0, 1, 0]])
actual_mask = model.predict([test_background])
numpy.testing.assert_array_almost_equal(expected_mask, actual_mask)