def _seq2seq_model(training=None): model = models.SequenceToSequence( inputters.WordEmbedder(16), inputters.WordEmbedder(16), encoders.SelfAttentionEncoder(2, 16, 4, 32), decoders.SelfAttentionDecoder(2, 16, 4, 32)) params = {} if training: params["optimizer"] = "SGD" params["learning_rate"] = 0.1 return model, params
def testInitializeWithDropoutOverride(self): model = models.SequenceToSequence( inputters.WordEmbedder(16), inputters.WordEmbedder(16), encoders.SelfAttentionEncoder(2, 16, 4, 32), decoders.SelfAttentionDecoder(2, 16, 4, 32)) self.assertEqual(model.encoder.dropout, 0.1) _, _, data_config = self._makeToyClassifierData() params = dict(dropout=0.3) model.initialize(data_config, params=params) self.assertEqual(model.encoder.dropout, 0.3)
def testSequenceToSequenceWithSharedEmbedding(self): model = models.SequenceToSequence( inputters.WordEmbedder(16), inputters.WordEmbedder(16), encoders.SelfAttentionEncoder(2, 16, 4, 32), decoders.SelfAttentionDecoder(2, 16, 4, 32), share_embeddings=models.EmbeddingsSharingLevel.ALL) _, _, data_config = self._makeToyEnDeData() data_config["target_vocabulary"] = data_config["source_vocabulary"] model.initialize(data_config) self.assertTrue(model.decoder.initialized) model.build(None) self.assertEqual(model.labels_inputter.embedding.ref(), model.decoder.output_layer.weight.ref())
def _seq2seq_model(training=None, shared_embeddings=False): model = models.SequenceToSequence( inputters.WordEmbedder(16), inputters.WordEmbedder(16), encoders.SelfAttentionEncoder(2, 16, 4, 32), decoders.SelfAttentionDecoder(2, 16, 4, 32), share_embeddings=( models.sequence_to_sequence.EmbeddingsSharingLevel.ALL if shared_embeddings else models.EmbeddingsSharingLevel.NONE), ) params = {} if training: params["optimizer"] = "SGD" params["learning_rate"] = 0.1 return model, params
def testSequenceToSequenceWithScheduledSampling(self): model = models.SequenceToSequence( inputters.WordEmbedder(16), inputters.WordEmbedder(16), encoders.SelfAttentionEncoder(2, 16, 4, 32), decoders.RNNDecoder(2, 16)) params = { "scheduled_sampling_type": "linear", "scheduled_sampling_read_probability": 0.8, "scheduled_sampling_k": 0.1 } features_file, labels_file, data_config = self._makeToyEnDeData() model.initialize(data_config, params=params) dataset = model.examples_inputter.make_training_dataset(features_file, labels_file, 16) features, labels = next(iter(dataset)) with self.assertRaises(ValueError): model(features, labels=labels, training=True) # step argument is required. outputs, _ = model(features, labels=labels, training=True, step=10) self.assertEqual(outputs["logits"].shape[1], labels["ids"].shape[1])