def _test_outputs(self, decoder, outputs, final_state, sequence_lengths, test_mode=False): # 4 trainable variables: cell-kernel, cell-bias, # fc-layer-weights, fc-layer-bias self.assertEqual(len(decoder.trainable_variables), 4) cell_dim = decoder.hparams.rnn_cell.kwargs.num_units with self.test_session() as sess: sess.run(tf.global_variables_initializer()) outputs_, final_state_, sequence_lengths_ = sess.run( [outputs, final_state, sequence_lengths], feed_dict={context.global_mode(): tf.estimator.ModeKeys.TRAIN}) self.assertIsInstance(outputs_, BasicRNNDecoderOutput) if not test_mode: self.assertEqual( outputs_.logits.shape, (self._batch_size, self._max_time, self._vocab_size)) self.assertEqual(outputs_.sample_id.shape, (self._batch_size, self._max_time)) np.testing.assert_array_equal( sequence_lengths_, [self._max_time] * self._batch_size) self.assertEqual(final_state_[0].shape, (self._batch_size, cell_dim))
def test_seq_pg_agent(self): """Tests logits. """ decoder = BasicRNNDecoder(vocab_size=self._vocab_size) outputs, _, sequence_length = decoder(decoding_strategy="infer_greedy", max_decoding_length=10, embedding=self._embedding, start_tokens=[1] * self._batch_size, end_token=2) agent = SeqPGAgent(outputs.sample_id, outputs.logits, sequence_length, decoder.trainable_variables) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) agent.sess = sess feed_dict = {context.global_mode(): tf.estimator.ModeKeys.TRAIN} for _ in range(2): vals = agent.get_samples(feed_dict=feed_dict) self.assertEqual(vals['samples'].shape[0], self._batch_size) loss_1 = agent.observe([1.] * self._batch_size) loss_2 = agent.observe([1.] * self._batch_size, train_policy=False) self.assertEqual(loss_1.shape, ()) self.assertEqual(loss_2.shape, ())
def _test_position_embedder(self, hparams): """Tests :class:`texar.modules.PositionEmbedder`. """ pos_size = 100 embedder = PositionEmbedder(position_size=pos_size, hparams=hparams) inputs = tf.ones([64, 16], dtype=tf.int32) outputs = embedder(inputs) emb_dim = embedder.dim if not isinstance(emb_dim, (list, tuple)): emb_dim = [emb_dim] hparams_dim = hparams["dim"] if not isinstance(hparams["dim"], (list, tuple)): hparams_dim = [hparams["dim"]] self.assertEqual(outputs.shape, [64, 16] + emb_dim) self.assertEqual(emb_dim, hparams_dim) self.assertEqual(embedder.position_size, 100) self.assertEqual(len(embedder.trainable_variables), 1) seq_length = tf.random_uniform([64], maxval=pos_size, dtype=tf.int32) outputs = embedder(sequence_length=seq_length) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) outputs_, max_seq_length = sess.run( [outputs, tf.reduce_max(seq_length)], feed_dict={global_mode(): tf.estimator.ModeKeys.TRAIN}) self.assertEqual(outputs_.shape, (64, max_seq_length) + tuple(emb_dim))
def test_get_rnn_cell(self): """Tests :func:`texar.core.layers.get_rnn_cell`. """ emb_dim = 4 num_units = 64 # Given instance hparams = {"type": rnn.LSTMCell(num_units)} cell = layers.get_rnn_cell(hparams) self.assertTrue(isinstance(cell, rnn.LSTMCell)) # Given class hparams = {"type": rnn.LSTMCell, "kwargs": {"num_units": 10}} cell = layers.get_rnn_cell(hparams) self.assertTrue(isinstance(cell, rnn.LSTMCell)) # Given string, and complex hyperparameters keep_prob_x = tf.placeholder(name='keep_prob', shape=[], dtype=tf.float32) hparams = { "type": "tensorflow.contrib.rnn.GRUCell", "kwargs": { "num_units": num_units }, "num_layers": 2, "dropout": { "input_keep_prob": 0.8, "state_keep_prob": keep_prob_x, "variational_recurrent": True, "input_size": [emb_dim, num_units] }, "residual": True, "highway": True } hparams_ = HParams(hparams, layers.default_rnn_cell_hparams()) cell = layers.get_rnn_cell(hparams_) batch_size = 16 inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32) output, state = cell(inputs, cell.zero_state(batch_size, dtype=tf.float32)) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) feed_dict = { keep_prob_x: 1.0, context.global_mode(): tf.estimator.ModeKeys.TRAIN } output_, state_ = sess.run([output, state], feed_dict=feed_dict) self.assertEqual(output_.shape[0], batch_size) if isinstance(state_, (list, tuple)): self.assertEqual(state_[0].shape[0], batch_size) self.assertEqual(state_[0].shape[1], hparams_.kwargs.num_units) else: self.assertEqual(state_.shape[0], batch_size) self.assertEqual(state_.shape[1], hparams_.kwargs.num_units)
def maybe_global_mode(mode): """Returns :func:`texar.contex.global_mode` if :attr:`mode` is `None`, otherwise returns :attr:`mode` as-is. """ if mode is None: return context.global_mode() else: return mode
def test_global_mode(self): """Tests the mode context manager. """ global_mode = context.global_mode() self.assertIsInstance(global_mode, tf.Tensor) mode_train = context.global_mode_train() mode_eval = context.global_mode_eval() mode_predict = context.global_mode_predict() with self.test_session() as sess: sess.run(tf.global_variables_initializer()) global_mode_ = sess.run(global_mode) self.assertEqual(tf.compat.as_str(global_mode_), tf.estimator.ModeKeys.TRAIN) global_mode_, mode_train_, mode_eval_, mode_predict_ = sess.run( [global_mode, mode_train, mode_eval, mode_predict], feed_dict={context.global_mode(): tf.estimator.ModeKeys.TRAIN}) self.assertEqual(global_mode_, tf.estimator.ModeKeys.TRAIN) self.assertTrue(mode_train_) self.assertFalse(mode_eval_) self.assertFalse(mode_predict_) global_mode_, mode_train_, mode_eval_, mode_predict_ = sess.run( [global_mode, mode_train, mode_eval, mode_predict], feed_dict={context.global_mode(): tf.estimator.ModeKeys.EVAL}) self.assertEqual(global_mode_, tf.estimator.ModeKeys.EVAL) self.assertFalse(mode_train_) self.assertTrue(mode_eval_) self.assertFalse(mode_predict_) global_mode_, mode_train_, mode_eval_, mode_predict_ = sess.run( [global_mode, mode_train, mode_eval, mode_predict], feed_dict={ context.global_mode(): tf.estimator.ModeKeys.PREDICT }) self.assertEqual(global_mode_, tf.estimator.ModeKeys.PREDICT) self.assertFalse(mode_train_) self.assertFalse(mode_eval_) self.assertTrue(mode_predict_) global_mode_values = tf.get_collection_ref(context._GLOBAL_MODE_KEY) self.assertEqual(len(global_mode_values), 1)
def test_decode_train(self): """Tests decoding in training mode. """ seq_length = np.random.randint(self._max_time, size=[self._batch_size ]) + 1 encoder_values_length = tf.constant(seq_length) hparams = { "attention": { "kwargs": { "num_units": self._attention_dim, # Note: to use sparsemax in TF-CPU, it looks # `memory_sequence_length` must equal max_time. #"probability_fn": "sparsemax" } } } decoder = AttentionRNNDecoder( memory=self._encoder_output, memory_sequence_length=encoder_values_length, vocab_size=self._vocab_size, hparams=hparams) helper_train = get_helper( decoder.hparams.helper_train.type, inputs=self._inputs, sequence_length=[self._max_time] * self._batch_size, **decoder.hparams.helper_train.kwargs.todict()) outputs, final_state, sequence_lengths = decoder(helper=helper_train) # 4+1 trainable variables: cell-kernel, cell-bias, # fc-weight, fc-bias, and # memory_layer: For LuongAttention, we only transform the memory layer; # thus num_units *must* match the expected query depth. self.assertEqual(len(decoder.trainable_variables), 5) cell_dim = decoder.hparams.rnn_cell.kwargs.num_units with self.test_session() as sess: sess.run(tf.global_variables_initializer()) outputs_, final_state_, sequence_lengths_ = sess.run( [outputs, final_state, sequence_lengths], feed_dict={context.global_mode(): tf.estimator.ModeKeys.TRAIN}) self.assertIsInstance(outputs_, AttentionRNNDecoderOutput) self.assertEqual( outputs_.logits.shape, (self._batch_size, self._max_time, self._vocab_size)) self.assertEqual(outputs_.sample_id.shape, (self._batch_size, self._max_time)) self.assertEqual(final_state_.cell_state[0].shape, (self._batch_size, cell_dim)) np.testing.assert_array_equal(sequence_lengths_, [self._max_time] * self._batch_size)
def test_decode_infer(self): """Tests decoding in inference mode. """ seq_length = np.random.randint(self._max_time, size=[self._batch_size ]) + 1 encoder_values_length = tf.constant(seq_length) hparams = { "attention": { "kwargs": { "num_units": 256, } } } decoder = AttentionRNNDecoder( vocab_size=self._vocab_size, memory=self._encoder_output, memory_sequence_length=encoder_values_length, hparams=hparams) helper_infer = get_helper( decoder.hparams.helper_infer.type, embedding=self._embedding, start_tokens=[1] * self._batch_size, end_token=2, **decoder.hparams.helper_train.kwargs.todict()) outputs, final_state, sequence_lengths = decoder(helper=helper_infer) # 4+1 trainable variables: cell-kernel, cell-bias, # fc-weight, fc-bias, and # memory_layer: For LuongAttention, we only transform the memory layer; # thus num_units *must* match the expected query depth. self.assertEqual(len(decoder.trainable_variables), 5) cell_dim = decoder.hparams.rnn_cell.kwargs.num_units with self.test_session() as sess: sess.run(tf.global_variables_initializer()) outputs_, final_state_, sequence_lengths_ = sess.run( [outputs, final_state, sequence_lengths], feed_dict={ context.global_mode(): tf.estimator.ModeKeys.PREDICT }) self.assertIsInstance(outputs_, AttentionRNNDecoderOutput) max_length = max(sequence_lengths_) self.assertEqual(outputs_.logits.shape, (self._batch_size, max_length, self._vocab_size)) self.assertEqual(outputs_.sample_id.shape, (self._batch_size, max_length)) self.assertEqual(final_state_.cell_state[0].shape, (self._batch_size, cell_dim))
def test_mode(self): """ Tests mode related utilities. """ training = mode.is_train_mode(None) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) training_ = sess.run(training) self.assertTrue(training_) training_ = sess.run( training, feed_dict={context.global_mode(): tf.estimator.ModeKeys.TRAIN}) self.assertTrue(training_) training_ = sess.run( training, feed_dict={context.global_mode(): tf.estimator.ModeKeys.EVAL}) self.assertFalse(training_) training = mode.is_train_mode(tf.estimator.ModeKeys.TRAIN) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) training_ = sess.run(training) self.assertTrue(training_)
def _test_word_embedder(self, hparams): """Tests :class:`texar.modules.WordEmbedder`. """ embedder = WordEmbedder(vocab_size=100, hparams=hparams) inputs = tf.ones([64, 16], dtype=tf.int32) outputs = embedder(inputs) inputs_soft = tf.ones([64, 16, embedder.vocab_size], dtype=tf.float32) outputs_soft = embedder(soft_ids=inputs_soft) emb_dim = embedder.dim if not isinstance(emb_dim, (list, tuple)): emb_dim = [emb_dim] hparams_dim = hparams["dim"] if not isinstance(hparams["dim"], (list, tuple)): hparams_dim = [hparams["dim"]] self.assertEqual(outputs.shape, [64, 16] + emb_dim) self.assertEqual(outputs_soft.shape, [64, 16] + emb_dim) self.assertEqual(emb_dim, hparams_dim) self.assertEqual(embedder.vocab_size, 100) self.assertEqual(len(embedder.trainable_variables), 1) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) outputs_, outputs_soft_ = sess.run( [outputs, outputs_soft], feed_dict={global_mode(): tf.estimator.ModeKeys.TRAIN}) self.assertEqual(outputs_.shape, (64, 16) + tuple(emb_dim)) self.assertEqual(outputs_soft_.shape, (64, 16) + tuple(emb_dim)) # Tests unknown input shapes inputs = tf.placeholder(dtype=tf.int64, shape=[None, None]) outputs = embedder(inputs) self.assertEqual(len(outputs.get_shape()), 2 + len(hparams_dim)) inputs_soft = tf.placeholder(dtype=tf.int64, shape=[None, None, None]) outputs_soft = embedder(soft_ids=inputs_soft) self.assertEqual(len(outputs_soft.get_shape()), 2 + len(hparams_dim))
def test_decode_infer(self): """Tests decoding in inferencee mode. """ output_layer = tf.layers.Dense(self._vocab_size) decoder = BasicRNNDecoder(vocab_size=self._vocab_size, output_layer=output_layer) helper_infer = get_helper( decoder.hparams.helper_infer.type, embedding=self._embedding, start_tokens=[self._vocab_size - 2] * self._batch_size, end_token=self._vocab_size - 1, **decoder.hparams.helper_train.kwargs.todict()) outputs, final_state, sequence_lengths = decoder(helper=helper_infer) # 4 trainable variables: embedding, cell-kernel, cell-bias, # fc-layer-weights, fc-layer-bias self.assertEqual(len(decoder.trainable_variables), 4) cell_dim = decoder.hparams.rnn_cell.kwargs.num_units with self.test_session() as sess: sess.run(tf.global_variables_initializer()) outputs_, final_state_, sequence_lengths_ = sess.run( [outputs, final_state, sequence_lengths], feed_dict={ context.global_mode(): tf.estimator.ModeKeys.PREDICT }) self.assertIsInstance(outputs_, BasicRNNDecoderOutput) max_length = max(sequence_lengths_) self.assertEqual(outputs_.logits.shape, (self._batch_size, max_length, self._vocab_size)) self.assertEqual(outputs_.sample_id.shape, (self._batch_size, max_length)) self.assertEqual(final_state_[0].shape, (self._batch_size, cell_dim))
def test_decode_train_with_tf(self): """Compares decoding results with TF built-in decoder. """ _inputs_placeholder = tf.placeholder( tf.int32, [self._batch_size, self._max_time], name="inputs") _embedding_placeholder = tf.placeholder( tf.float32, [self._vocab_size, self._emb_dim], name="emb") inputs = tf.nn.embedding_lookup(_embedding_placeholder, _inputs_placeholder) output_layer = tf.layers.Dense(self._vocab_size) decoder = BasicRNNDecoder(vocab_size=self._vocab_size, output_layer=output_layer) helper_train = get_helper( decoder.hparams.helper_train.type, inputs=inputs, sequence_length=[self._max_time] * self._batch_size, **decoder.hparams.helper_train.kwargs.todict()) outputs, final_state, sequence_lengths = decoder(helper=helper_train) tf_helper = tf.contrib.seq2seq.TrainingHelper( inputs, [self._max_time] * self._batch_size) tf_decoder = tf.contrib.seq2seq.BasicDecoder(decoder.cell, tf_helper, decoder.cell.zero_state( self._batch_size, tf.float32), output_layer=output_layer) tf_outputs, tf_final_state, tf_sequence_lengths = \ tf.contrib.seq2seq.dynamic_decode(tf_decoder) cell_dim = decoder.hparams.rnn_cell.kwargs.num_units with self.test_session() as sess: sess.run(tf.global_variables_initializer()) inputs_ = np.random.randint(self._vocab_size, size=(self._batch_size, self._max_time), dtype=np.int32) embedding_ = np.random.randn(self._vocab_size, self._emb_dim) outputs_, final_state_, sequence_lengths_ = sess.run( [outputs, final_state, sequence_lengths], feed_dict={ context.global_mode(): tf.estimator.ModeKeys.TRAIN, _inputs_placeholder: inputs_, _embedding_placeholder: embedding_ }) self.assertEqual(final_state_[0].shape, (self._batch_size, cell_dim)) tf_outputs_, tf_final_state_, tf_sequence_lengths_ = sess.run( [tf_outputs, tf_final_state, tf_sequence_lengths], feed_dict={ context.global_mode(): tf.estimator.ModeKeys.TRAIN, _inputs_placeholder: inputs_, _embedding_placeholder: embedding_ }) np.testing.assert_array_equal(outputs_.logits, tf_outputs_.rnn_output) np.testing.assert_array_equal(outputs_.sample_id, tf_outputs_.sample_id) np.testing.assert_array_equal(final_state_.c, tf_final_state_.c) np.testing.assert_array_equal(final_state_.h, tf_final_state_.h) np.testing.assert_array_equal(sequence_lengths_, tf_sequence_lengths_)
def _test_beam_search(self, decoder, initial_state=None, tiled_initial_state=None, tf_initial_state=None, beam_width_1=1, initiated=False): ## Compare with tf built-in BeamSearchDecoder outputs, final_state, _ = beam_search_decode(decoder_or_cell=decoder, embedding=self._embedding, start_tokens=[1] * self._batch_size, end_token=2, beam_width=beam_width_1, max_decoding_length=20) self.assertIsInstance(outputs, tf.contrib.seq2seq.FinalBeamSearchDecoderOutput) self.assertIsInstance(final_state, tf.contrib.seq2seq.BeamSearchDecoderState) num_trainable_variables = len(tf.trainable_variables()) _ = decoder(decoding_strategy='infer_greedy', embedding=self._embedding, start_tokens=[1] * self._batch_size, end_token=2, max_decoding_length=20) self.assertEqual(num_trainable_variables, len(tf.trainable_variables())) if tf_initial_state is None: tf_initial_state = decoder.cell.zero_state( self._batch_size * beam_width_1, tf.float32) beam_decoder = BeamSearchDecoder(cell=decoder.cell, embedding=self._embedding, start_tokens=[1] * self._batch_size, end_token=2, initial_state=tf_initial_state, beam_width=beam_width_1, output_layer=decoder.output_layer) outputs_1, final_state_1, _ = dynamic_decode(decoder=beam_decoder, maximum_iterations=20) ## Tests time major outputs_2, _, _ = beam_search_decode( decoder_or_cell=decoder, embedding=self._embedding, start_tokens=[1] * self._batch_size, end_token=2, beam_width=self._beam_width, initial_state=initial_state, tiled_initial_state=tiled_initial_state, max_decoding_length=21) outputs_3, _, _ = beam_search_decode( decoder_or_cell=decoder, embedding=self._embedding, start_tokens=[1] * self._batch_size, end_token=2, beam_width=self._beam_width, initial_state=initial_state, tiled_initial_state=tiled_initial_state, max_decoding_length=21, output_time_major=True) with self.test_session() as sess: if not initiated: sess.run(tf.global_variables_initializer()) outputs_, final_state_, outputs_1_, final_state_1_ = sess.run( [outputs, final_state, outputs_1, final_state_1], feed_dict={ context.global_mode(): tf.estimator.ModeKeys.PREDICT }) np.testing.assert_array_equal(outputs_.predicted_ids, outputs_1_.predicted_ids) np.testing.assert_array_equal( outputs_.beam_search_decoder_output.scores, outputs_1_.beam_search_decoder_output.scores) np.testing.assert_array_equal( outputs_.beam_search_decoder_output.predicted_ids, outputs_1_.beam_search_decoder_output.predicted_ids) np.testing.assert_array_equal( outputs_.beam_search_decoder_output.parent_ids, outputs_1_.beam_search_decoder_output.parent_ids) np.testing.assert_array_equal(final_state_.log_probs, final_state_1_.log_probs) np.testing.assert_array_equal(final_state_.lengths, final_state_1_.lengths) outputs_2_, outputs_3_ = sess.run([outputs_2, outputs_3], feed_dict={ context.global_mode(): tf.estimator.ModeKeys.PREDICT }) self.assertEqual(outputs_2_.predicted_ids.shape, tuple([self._batch_size, 21, 11])) self.assertEqual(outputs_3_.predicted_ids.shape, tuple([21, self._batch_size, 11]))