def testTrainLanguageModel(self): src = test_util.make_data_file( os.path.join(self.get_temp_dir(), "src.txt"), ["1 2 3 4", "5 6 7 8 9", "3 2"]) vocab = test_util.make_vocab( os.path.join(self.get_temp_dir(), "vocab.txt"), list(map(str, range(10)))) config = { "data": { "train_features_file": src, "vocabulary": vocab, }, "params": { "learning_rate": 0.0005, "optimizer": "Adam" }, "train": { "batch_size": 10, "max_step": 2, }, } model = models.LanguageModel(decoders.SelfAttentionDecoder( 2, num_units=32, ffn_inner_dim=32), embedding_size=16, reuse_embedding=False) runner = Runner(model, config) runner.train()
def testCreateVariablesLanguageModel(self): _, data_config = self._makeToyLMData() decoder = decoders.SelfAttentionDecoder( 2, num_units=16, num_heads=4, ffn_inner_dim=32, num_sources=0) model = models.LanguageModel(decoder, embedding_size=16) model.initialize(data_config) model.create_variables() self.assertTrue(len(model.trainable_variables) > 0)
def testLanguageModelWithMissingStart(self): _, data_config = self._makeToyLMData() decoder = decoders.SelfAttentionDecoder( 2, num_units=16, num_heads=4, ffn_inner_dim=32, num_sources=0) model = models.LanguageModel(decoder, embedding_size=16) model.initialize(data_config) features, _ = model.features_inputter.make_features(tf.constant("")) with self.assertRaises(tf.errors.InvalidArgumentError): model(features)
def testLanguageModelServing(self): _, data_config = self._makeToyLMData() decoder = decoders.SelfAttentionDecoder(2, num_units=16, num_heads=4, ffn_inner_dim=32, num_sources=0) model = models.LanguageModel(decoder, embedding_size=16) model.initialize(data_config) function = model.serve_function() function.get_concrete_function()
def testLanguageModelWithStartOfSentence(self): _, data_config = self._makeToyLMData() data_config["sequence_controls"] = dict(start=True, end=False) decoder = decoders.SelfAttentionDecoder( 2, num_units=16, num_heads=4, ffn_inner_dim=32, num_sources=0) model = models.LanguageModel(decoder, embedding_size=16) model.initialize(data_config, params={"maximum_decoding_length": 1}) features, _ = model.features_inputter.make_features(tf.constant("")) features = tf.nest.map_structure(lambda t: tf.expand_dims(t, 0), features) # Add batch dim. _, predictions = self.evaluate(model(features)) # Predictions should not include the leading <s>. self.assertEqual(predictions["length"][0], 1) self.assertTupleEqual(predictions["tokens"].shape, (1, 1))
def testLanguageModel(self, mode): # Mainly test that the code does not throw. decoder = decoders.SelfAttentionDecoder( 2, num_units=16, num_heads=4, ffn_inner_dim=32, num_sources=0) model = models.LanguageModel(decoder, embedding_size=16) features_file, data_config = self._makeToyLMData() params = { "optimizer": "SGD", "learning_rate": 0.1} self._testGenericModel( model, mode, features_file, data_config=data_config, batch_size=1 if mode == tf.estimator.ModeKeys.PREDICT else 16, prediction_heads=["tokens", "length"], params=params)