def _seq2seq_model(training=None):
model = models.SequenceToSequence(
inputters.WordEmbedder(16), inputters.WordEmbedder(16),
encoders.SelfAttentionEncoder(2, 16, 4, 32),
decoders.SelfAttentionDecoder(2, 16, 4, 32))
params = {}
if training:
params["optimizer"] = "SGD"
params["learning_rate"] = 0.1
return model, params
def testInitializeWithDropoutOverride(self):
model = models.SequenceToSequence(
inputters.WordEmbedder(16), inputters.WordEmbedder(16),
encoders.SelfAttentionEncoder(2, 16, 4, 32),
decoders.SelfAttentionDecoder(2, 16, 4, 32))
self.assertEqual(model.encoder.dropout, 0.1)
_, _, data_config = self._makeToyClassifierData()
params = dict(dropout=0.3)
model.initialize(data_config, params=params)
self.assertEqual(model.encoder.dropout, 0.3)
def testSequenceToSequenceWithSharedEmbedding(self):
model = models.SequenceToSequence(
inputters.WordEmbedder(16),
inputters.WordEmbedder(16),
encoders.SelfAttentionEncoder(2, 16, 4, 32),
decoders.SelfAttentionDecoder(2, 16, 4, 32),
share_embeddings=models.EmbeddingsSharingLevel.ALL)
_, _, data_config = self._makeToyEnDeData()
data_config["target_vocabulary"] = data_config["source_vocabulary"]
model.initialize(data_config)
self.assertTrue(model.decoder.initialized)
model.build(None)
self.assertEqual(model.labels_inputter.embedding.ref(),
model.decoder.output_layer.weight.ref())
def _seq2seq_model(training=None, shared_embeddings=False):
model = models.SequenceToSequence(
inputters.WordEmbedder(16),
inputters.WordEmbedder(16),
encoders.SelfAttentionEncoder(2, 16, 4, 32),
decoders.SelfAttentionDecoder(2, 16, 4, 32),
share_embeddings=(
models.sequence_to_sequence.EmbeddingsSharingLevel.ALL
if shared_embeddings else models.EmbeddingsSharingLevel.NONE),
)
params = {}
if training:
params["optimizer"] = "SGD"
params["learning_rate"] = 0.1
return model, params
def testSequenceToSequenceWithScheduledSampling(self):
model = models.SequenceToSequence(
inputters.WordEmbedder(16),
inputters.WordEmbedder(16),
encoders.SelfAttentionEncoder(2, 16, 4, 32),
decoders.RNNDecoder(2, 16))
params = {
"scheduled_sampling_type": "linear",
"scheduled_sampling_read_probability": 0.8,
"scheduled_sampling_k": 0.1
}
features_file, labels_file, data_config = self._makeToyEnDeData()
model.initialize(data_config, params=params)
dataset = model.examples_inputter.make_training_dataset(features_file, labels_file, 16)
features, labels = next(iter(dataset))
with self.assertRaises(ValueError):
model(features, labels=labels, training=True) # step argument is required.
outputs, _ = model(features, labels=labels, training=True, step=10)
self.assertEqual(outputs["logits"].shape[1], labels["ids"].shape[1])
