def test_mask_is_computed_correctly(self):
background_input = Input(shape=(None, 3), dtype='int32')
embedding = TimeDistributedEmbedding(input_dim=3,
output_dim=2,
mask_zero=True)
embedded_background = embedding(background_input)
encoded_background = BOWEncoder(units=2)(embedded_background)
encoded_background_with_mask = AddEncoderMask()(
[encoded_background, embedded_background])
mask_output = OutputMask()(encoded_background_with_mask)
model = DeepQaModel(inputs=[background_input], outputs=mask_output)
test_background = numpy.asarray([[
[0, 0, 0],
[2, 2, 2],
[0, 0, 0],
[0, 1, 2],
[1, 0, 0],
[0, 0, 0],
[0, 1, 0],
[1, 1, 1],
]])
expected_mask = numpy.asarray([[0, 1, 0, 1, 1, 0, 1, 1]])
actual_mask = model.predict([test_background])
numpy.testing.assert_array_equal(expected_mask, actual_mask)
def test_on_masked_input(self):
# TODO(matt): I don't really like having to build the whole model up to the attention
# component here, but I'm not sure how to just test the selector with the right mask
# without going through this.
sentence_input = Input(shape=(3, ), dtype='int32')
background_input = Input(shape=(3, 3), dtype='int32')
embedding = TimeDistributedEmbedding(input_dim=3,
output_dim=2,
mask_zero=True)
embedded_sentence = embedding(sentence_input)
embedded_background = embedding(background_input)
encoder = BOWEncoder(output_dim=2)
encoded_sentence = encoder(embedded_sentence)
encoded_background = EncoderWrapper(encoder)(embedded_background)
merge_mode = lambda layer_outs: K.concatenate([
K.expand_dims(layer_outs[0], dim=1),
K.expand_dims(layer_outs[0], dim=1), layer_outs[1]
],
axis=1)
merge_masks = lambda mask_outs: K.concatenate([
K.expand_dims(K.zeros_like(mask_outs[1][:, 0]), dim=1),
K.expand_dims(K.zeros_like(mask_outs[1][:, 0]), dim=1), mask_outs[1
]
],
axis=1)
merged = merge([encoded_sentence, encoded_background],
mode=merge_mode,
output_shape=(5, 2),
output_mask=merge_masks)
merged_mask = OutputMask()(merged)
selector = DotProductKnowledgeSelector()
attention_weights = selector(merged)
model = DeepQaModel(input=[sentence_input, background_input],
output=[merged_mask, attention_weights])
model.summary(show_masks=True)
test_input = numpy.asarray([[2, 2, 2]])
test_background = numpy.asarray([[
[2, 2, 2],
[2, 2, 2],
[0, 0, 0],
]])
expected_mask = numpy.asarray([[0, 0, 1, 1, 0]])
expected_attention = numpy.asarray([[0.5, 0.5, 0.0]])
actual_mask, actual_attention = model.predict(
[test_input, test_background])
numpy.testing.assert_array_almost_equal(expected_mask, actual_mask)
numpy.testing.assert_array_almost_equal(expected_attention,
actual_attention)
def test_handles_multiple_masks(self):
# We'll use the SlotSimilarityTupleMatcher to test this, because it takes two masked
# inputs. Here we're using an input of shape (batch_size, num_options, num_tuples,
# num_slots, num_words).
tuple_input = Input(shape=(2, 3, 4, 5), dtype='int32')
tuple_input_2 = Input(shape=(2, 3, 4, 5), dtype='int32')
embedding = TimeDistributedEmbedding(input_dim=3,
output_dim=6,
mask_zero=True)
# shape is now (batch_size, num_options, num_tuples, num_slots, num_words, embedding_dim)
embedded_tuple = embedding(tuple_input)
embedded_tuple_2 = embedding(tuple_input_2)
encoder = EncoderWrapper(EncoderWrapper(EncoderWrapper(BOWEncoder())))
# shape is now (batch_size, num_options, num_tuples, num_slots, embedding_dim)
encoded_tuple = encoder(embedded_tuple)
encoded_tuple_2 = encoder(embedded_tuple_2)
# Shape of input to the tuple matcher is [(batch size, 2, 3, 4, 6), (batch size, 2, 3, 4, 6)]
# Shape of input_mask to the tuple matcher is [(batch size, 2, 3, 4), (batch size, 2, 3, 4)]
# Expected output mask shape (batch_size, 2, 3)
time_distributed = TimeDistributedWithMask(
TimeDistributedWithMask(
SlotSimilarityTupleMatcher({"type": "cosine_similarity"})))
time_distributed_output = time_distributed(
[encoded_tuple, encoded_tuple_2])
mask_output = OutputMask()(time_distributed_output)
model = DeepQaModel(input=[tuple_input, tuple_input_2],
output=mask_output)
zeros = [0, 0, 0, 0, 0]
non_zeros = [1, 1, 1, 1, 1]
# shape: (batch size, num_options, num_tuples, num_slots, num_words), or (1, 2, 3, 4, 5)
tuples1 = numpy.asarray([[[[zeros, zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[non_zeros, non_zeros, zeros, zeros]],
[[non_zeros, non_zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[zeros, zeros, zeros, zeros]]]])
tuples2 = numpy.asarray([[[[non_zeros, zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[zeros, zeros, zeros, zeros]],
[[non_zeros, non_zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros],
[non_zeros, zeros, zeros, zeros]]]])
actual_mask = model.predict([tuples1, tuples2])
expected_mask = numpy.asarray(
[[[0, 1, 0], [1, 1,
0]]]) # shape: (batch size, num_options, num_tuples)
assert actual_mask.shape == (1, 2, 3)
numpy.testing.assert_array_almost_equal(expected_mask, actual_mask)
def test_on_all_zeros(self):
sentence_length = 5
embedding_size = 10
vocabulary_size = 15
input_layer = Input(shape=(sentence_length,), dtype='int32')
# Embedding masks zeros
embedding = Embedding(input_dim=vocabulary_size, output_dim=embedding_size, mask_zero=True)
encoder = BOWEncoder()
embedded_input = embedding(input_layer)
encoded_input = encoder(embedded_input)
model = Model(input=input_layer, output=encoded_input)
model.compile(loss="mse", optimizer="sgd") # Will not train this model
test_input = numpy.asarray([[0, 0, 0, 0, 0]], dtype='int32')
# Omitting the first element (0), because that is supposed to be masked in the model.
expected_output = numpy.zeros((1, embedding_size))
actual_output = model.predict(test_input)
# Following comparison is till the sixth decimal.
numpy.testing.assert_array_almost_equal(expected_output, actual_output)
def test_on_unmasked_input(self):
sentence_length = 5
embedding_size = 10
vocabulary_size = 15
input_layer = Input(shape=(sentence_length,), dtype='int32')
# Embedding does not mask zeros
embedding = Embedding(input_dim=vocabulary_size, output_dim=embedding_size)
encoder = BOWEncoder()
embedded_input = embedding(input_layer)
encoded_input = encoder(embedded_input)
model = Model(input=input_layer, output=encoded_input)
model.compile(loss="mse", optimizer="sgd") # Will not train this model
test_input = numpy.asarray([[0, 3, 1, 7, 10]], dtype='int32')
embedding_weights = embedding.get_weights()[0] # get_weights returns a list with one element.
expected_output = numpy.mean(embedding_weights[test_input], axis=1)
actual_output = model.predict(test_input)
# Exact comparison of the two arrays may break because theano's floating point operations
# usually have an epsilon. The following comparison is done till the sixth decimal, hence good enough.
numpy.testing.assert_array_almost_equal(expected_output, actual_output)
def test_on_unmasked_input(self):
sentence_length = 5
embedding_dim = 10
vocabulary_size = 15
input_layer = Input(shape=(sentence_length, ), dtype='int32')
# Embedding does not mask zeros
embedding = Embedding(input_dim=vocabulary_size,
output_dim=embedding_dim)
encoder = BOWEncoder()
embedded_input = embedding(input_layer)
encoded_input = encoder(embedded_input)
model = Model(inputs=input_layer, outputs=encoded_input)
model.compile(loss="mse", optimizer="sgd") # Will not train this model
test_input = numpy.asarray([[0, 3, 1, 7, 10]], dtype='int32')
embedding_weights = embedding.get_weights()[
0] # get_weights returns a list with one element.
expected_output = numpy.mean(embedding_weights[test_input], axis=1)
actual_output = model.predict(test_input)
numpy.testing.assert_array_almost_equal(expected_output, actual_output)
def test_mask_is_computed_correctly(self):
background_input = Input(shape=(3, 3), dtype='int32')
embedding = Embedding(input_dim=3, output_dim=2, mask_zero=True)
embedded_background = embedding(background_input)
encoded_background = EncoderWrapper(BOWEncoder(units=2))(embedded_background)
mask_output = OutputMask()(encoded_background)
model = DeepQaModel(inputs=[background_input], outputs=mask_output)
test_background = numpy.asarray([
[
[0, 0, 0],
[2, 2, 2],
[0, 0, 0],
]
])
expected_mask = numpy.asarray([[0, 1, 0]])
actual_mask = model.predict([test_background])
numpy.testing.assert_array_almost_equal(expected_mask, actual_mask)
def test_on_masked_input(self):
sentence_length = 5
embedding_dim = 10
vocabulary_size = 15
input_layer = Input(shape=(sentence_length,), dtype='int32')
# Embedding masks zeros
embedding = Embedding(input_dim=vocabulary_size, output_dim=embedding_dim, mask_zero=True)
encoder = BOWEncoder()
embedded_input = embedding(input_layer)
encoded_input = encoder(embedded_input)
model = Model(inputs=input_layer, outputs=encoded_input)
model.compile(loss="mse", optimizer="sgd") # Will not train this model
test_input = numpy.asarray([[0, 3, 1, 7, 10]], dtype='int32')
embedding_weights = embedding.get_weights()[0] # get_weights returns a list with one element.
# Omitting the first element (0), because that is supposed to be masked in the model.
expected_output = numpy.mean(embedding_weights[test_input[:, 1:]], axis=1)
actual_output = model.predict(test_input)
# Following comparison is till the sixth decimal.
numpy.testing.assert_array_almost_equal(expected_output, actual_output)
def test_mask_is_computed_correctly(self):
# TODO(matt): I don't really like having to build a model to test this, but I'm not sure of
# how else to do it.
background_input = Input(shape=(3, 3), dtype='int32')
embedding = TimeDistributedEmbedding(input_dim=3,
output_dim=2,
mask_zero=True)
embedded_background = embedding(background_input)
encoded_background = EncoderWrapper(
BOWEncoder(output_dim=2))(embedded_background)
mask_output = OutputMask()(encoded_background)
model = DeepQaModel(input=[background_input], output=mask_output)
test_background = numpy.asarray([[
[0, 0, 0],
[2, 2, 2],
[0, 0, 0],
]])
expected_mask = numpy.asarray([[0, 1, 0]])
actual_mask = model.predict([test_background])
numpy.testing.assert_array_almost_equal(expected_mask, actual_mask)
def _build_model(self):
# Input: (slots, query-slot)
# Output: (queried-slot-phrase-embedding)
# Step 1: Convert the frame input into sequences of word vectors
# corresponding to the slot values/phrases (ignoring the slot names).
# slots_input: numpy array: int32 (batch_size, num_slots, text_length).
# Left padded arrays of word indices from sentences in training data.
# We have a list of phrases as input. The base class implementation of
# _get_sentence_shape provides sentence length, which is the phrase length
# in this model. We shall additionally supply number of slots.
slots_input = Input(
shape=((self.num_slots, ) +
self._get_sentence_shape()), # Note: excludes batch size
dtype='int32', # shape encodes lengths, lengths are of type int.
name="slots_input" # Should it be "words"?
)
# Step 2: Pass the sequences of word vectors through the sentence encoder to get a sentence vector.
# Shape: (batch_size, number_of_slots, max_phrase_len, embedding_dim)
each_slot_embedding = self._embed_input(slots_input)
# average out over phrase_len:
# from: batch_size, number_of_slots, phrase_len, embedding_dim
# output should become: batch_size, number_of_slots, embedding_dim
averaging_layer = AveragedBOWEncoder(2, 4)
# batch_size, number_of_slots, embedding_dim
each_slot_embedding = averaging_layer(each_slot_embedding)
# Shape: (batch_size, number_of_slots, embedding_dim)
# We first convert a sentence to a sequence of word embeddings
# and then apply a stack of seq2seq encoders.
for i in range(self.num_stacked_rnns):
encoder = self._get_seq2seq_encoder(
name="encoder_{}".format(i),
fallback_behavior="use default params")
# shape still (batch_size, number_of_slots, 2 * embedding_dim)
each_slot_embedding = encoder(each_slot_embedding)
# From (batch_size, number_of_slots, 2 * embedding_dim),
# convert to batch_size, 2*embedding_dim
bow_features = BOWEncoder()
avg_slot_embedding = bow_features(each_slot_embedding)
# Add a dropout after LSTM.
regularized_embedding = Dropout(0.2)(avg_slot_embedding)
# Step 3: Dense projection
# From:(batch_size, 2*embedding_dim),
# convert to (batch_size, nn_embedding_dim),
# so, a dense layer is needed
projection_layer = Dense(int(self.nearest_neighbor_dim),
activation='relu',
name='projector')
projected_frame = projection_layer(regularized_embedding)
# Step 4: Define squared loss against labels as the loss.
# Requires that training input contain a vector representation of the queried slot as label.
# Further, we need to find all the possible nearest neighbors for this vector.
return DeepQaModel(inputs=slots_input, outputs=projected_frame)