def testSelfAttentionEncoder(self, dtype): tf.keras.backend.set_floatx(dtype.name) encoder = encoders.SelfAttentionEncoder( 3, num_units=20, num_heads=4, ffn_inner_dim=40) inputs = tf.random.uniform([4, 5, 10], dtype=dtype) lengths = tf.constant([4, 3, 5, 2]) outputs, _, _ = encoder(inputs, sequence_length=lengths, training=True) self.assertListEqual(outputs.shape.as_list(), [4, 5, 20]) self.assertEqual(outputs.dtype, dtype) tf.keras.backend.set_floatx("float32")
def testInitializeWithDropoutOverride(self): model = models.SequenceToSequence( inputters.WordEmbedder(16), inputters.WordEmbedder(16), encoders.SelfAttentionEncoder(2, 16, 4, 32), decoders.SelfAttentionDecoder(2, 16, 4, 32)) self.assertEqual(model.encoder.dropout, 0.1) _, _, data_config = self._makeToyClassifierData() params = dict(dropout=0.3) model.initialize(data_config, params=params) self.assertEqual(model.encoder.dropout, 0.3)
def _seq2seq_model(training=None): model = models.SequenceToSequence( inputters.WordEmbedder(16), inputters.WordEmbedder(16), encoders.SelfAttentionEncoder(2, 16, 4, 32), decoders.SelfAttentionDecoder(2, 16, 4, 32)) params = {} if training: params["optimizer"] = "SGD" params["learning_rate"] = 0.1 return model, params
def testSequenceClassifierWithSelfAttentionEncoder(self): # SelfAttentionEncoder does not return a state, so test that the classifier # does not crash on this. model = models.SequenceClassifier( inputters.WordEmbedder(10), encoders.SelfAttentionEncoder(num_layers=2, num_units=16, num_heads=4, ffn_inner_dim=32)) features_file, labels_file, data_config = self._makeToyClassifierData() model.initialize(data_config) dataset = model.examples_inputter.make_training_dataset(features_file, labels_file, 16) features, labels = iter(dataset).next() model(features, labels, training=True)
def testSequenceToSequenceWithSharedEmbedding(self): model = models.SequenceToSequence( inputters.WordEmbedder(16), inputters.WordEmbedder(16), encoders.SelfAttentionEncoder(2, 16, 4, 32), decoders.SelfAttentionDecoder(2, 16, 4, 32), share_embeddings=models.EmbeddingsSharingLevel.ALL) _, _, data_config = self._makeToyEnDeData() data_config["target_vocabulary"] = data_config["source_vocabulary"] model.initialize(data_config) self.assertTrue(model.decoder.initialized) model.build(None) self.assertEqual(model.labels_inputter.embedding.ref(), model.decoder.output_layer.weight.ref())
def _seq2seq_model(training=None, shared_embeddings=False): model = models.SequenceToSequence( inputters.WordEmbedder(16), inputters.WordEmbedder(16), encoders.SelfAttentionEncoder(2, 16, 4, 32), decoders.SelfAttentionDecoder(2, 16, 4, 32), share_embeddings=( models.sequence_to_sequence.EmbeddingsSharingLevel.ALL if shared_embeddings else models.EmbeddingsSharingLevel.NONE), ) params = {} if training: params["optimizer"] = "SGD" params["learning_rate"] = 0.1 return model, params
def testSelfAttentionEncoder(self): sequence_length = [17, 21, 20] inputs = _build_dummy_sequences(sequence_length, depth=10) encoder = encoders.SelfAttentionEncoder( 3, num_units=36, num_heads=4, ffn_inner_dim=52) outputs, state, encoded_length = encoder.encode( inputs, sequence_length=tf.constant(sequence_length)) self.assertEqual(3, len(state)) for s in state: self.assertIsInstance(s, tf.Tensor) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) outputs, encoded_length = sess.run([outputs, encoded_length]) self.assertAllEqual([3, 21, 36], outputs.shape) self.assertAllEqual(sequence_length, encoded_length)
def testSequenceToSequenceWithScheduledSampling(self): model = models.SequenceToSequence( inputters.WordEmbedder(16), inputters.WordEmbedder(16), encoders.SelfAttentionEncoder(2, 16, 4, 32), decoders.RNNDecoder(2, 16)) params = { "scheduled_sampling_type": "linear", "scheduled_sampling_read_probability": 0.8, "scheduled_sampling_k": 0.1 } features_file, labels_file, data_config = self._makeToyEnDeData() model.initialize(data_config, params=params) dataset = model.examples_inputter.make_training_dataset(features_file, labels_file, 16) features, labels = next(iter(dataset)) with self.assertRaises(ValueError): model(features, labels=labels, training=True) # step argument is required. outputs, _ = model(features, labels=labels, training=True, step=10) self.assertEqual(outputs["logits"].shape[1], labels["ids"].shape[1])