def testSelfAttentionEncoder(self, dtype):
tf.keras.backend.set_floatx(dtype.name)
encoder = encoders.SelfAttentionEncoder(
3, num_units=20, num_heads=4, ffn_inner_dim=40)
inputs = tf.random.uniform([4, 5, 10], dtype=dtype)
lengths = tf.constant([4, 3, 5, 2])
outputs, _, _ = encoder(inputs, sequence_length=lengths, training=True)
self.assertListEqual(outputs.shape.as_list(), [4, 5, 20])
self.assertEqual(outputs.dtype, dtype)
tf.keras.backend.set_floatx("float32")
def testInitializeWithDropoutOverride(self):
model = models.SequenceToSequence(
inputters.WordEmbedder(16), inputters.WordEmbedder(16),
encoders.SelfAttentionEncoder(2, 16, 4, 32),
decoders.SelfAttentionDecoder(2, 16, 4, 32))
self.assertEqual(model.encoder.dropout, 0.1)
_, _, data_config = self._makeToyClassifierData()
params = dict(dropout=0.3)
model.initialize(data_config, params=params)
self.assertEqual(model.encoder.dropout, 0.3)
def _seq2seq_model(training=None):
model = models.SequenceToSequence(
inputters.WordEmbedder(16), inputters.WordEmbedder(16),
encoders.SelfAttentionEncoder(2, 16, 4, 32),
decoders.SelfAttentionDecoder(2, 16, 4, 32))
params = {}
if training:
params["optimizer"] = "SGD"
params["learning_rate"] = 0.1
return model, params
def testSequenceClassifierWithSelfAttentionEncoder(self):
# SelfAttentionEncoder does not return a state, so test that the classifier
# does not crash on this.
model = models.SequenceClassifier(
inputters.WordEmbedder(10),
encoders.SelfAttentionEncoder(num_layers=2, num_units=16, num_heads=4, ffn_inner_dim=32))
features_file, labels_file, data_config = self._makeToyClassifierData()
model.initialize(data_config)
dataset = model.examples_inputter.make_training_dataset(features_file, labels_file, 16)
features, labels = iter(dataset).next()
model(features, labels, training=True)
def testSequenceToSequenceWithSharedEmbedding(self):
model = models.SequenceToSequence(
inputters.WordEmbedder(16),
inputters.WordEmbedder(16),
encoders.SelfAttentionEncoder(2, 16, 4, 32),
decoders.SelfAttentionDecoder(2, 16, 4, 32),
share_embeddings=models.EmbeddingsSharingLevel.ALL)
_, _, data_config = self._makeToyEnDeData()
data_config["target_vocabulary"] = data_config["source_vocabulary"]
model.initialize(data_config)
self.assertTrue(model.decoder.initialized)
model.build(None)
self.assertEqual(model.labels_inputter.embedding.ref(),
model.decoder.output_layer.weight.ref())
def _seq2seq_model(training=None, shared_embeddings=False):
model = models.SequenceToSequence(
inputters.WordEmbedder(16),
inputters.WordEmbedder(16),
encoders.SelfAttentionEncoder(2, 16, 4, 32),
decoders.SelfAttentionDecoder(2, 16, 4, 32),
share_embeddings=(
models.sequence_to_sequence.EmbeddingsSharingLevel.ALL
if shared_embeddings else models.EmbeddingsSharingLevel.NONE),
)
params = {}
if training:
params["optimizer"] = "SGD"
params["learning_rate"] = 0.1
return model, params
def testSelfAttentionEncoder(self):
sequence_length = [17, 21, 20]
inputs = _build_dummy_sequences(sequence_length, depth=10)
encoder = encoders.SelfAttentionEncoder(
3, num_units=36, num_heads=4, ffn_inner_dim=52)
outputs, state, encoded_length = encoder.encode(
inputs, sequence_length=tf.constant(sequence_length))
self.assertEqual(3, len(state))
for s in state:
self.assertIsInstance(s, tf.Tensor)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
outputs, encoded_length = sess.run([outputs, encoded_length])
self.assertAllEqual([3, 21, 36], outputs.shape)
self.assertAllEqual(sequence_length, encoded_length)
def testSequenceToSequenceWithScheduledSampling(self):
model = models.SequenceToSequence(
inputters.WordEmbedder(16),
inputters.WordEmbedder(16),
encoders.SelfAttentionEncoder(2, 16, 4, 32),
decoders.RNNDecoder(2, 16))
params = {
"scheduled_sampling_type": "linear",
"scheduled_sampling_read_probability": 0.8,
"scheduled_sampling_k": 0.1
}
features_file, labels_file, data_config = self._makeToyEnDeData()
model.initialize(data_config, params=params)
dataset = model.examples_inputter.make_training_dataset(features_file, labels_file, 16)
features, labels = next(iter(dataset))
with self.assertRaises(ValueError):
model(features, labels=labels, training=True) # step argument is required.
outputs, _ = model(features, labels=labels, training=True, step=10)
self.assertEqual(outputs["logits"].shape[1], labels["ids"].shape[1])
