def _testDecoderBeamSearchDecodeHelperWithOutput(self, params, src_seq_len=None, src_enc_padding=None): config = tf.ConfigProto(graph_options=tf.GraphOptions( optimizer_options=tf.OptimizerOptions(do_function_inlining=False))) p = params with self.session(use_gpu=False, config=config) as sess, self.SetEval(True): tf.set_random_seed(837274904) np.random.seed(837575) p.beam_search.num_hyps_per_beam = 4 p.dtype = tf.float32 p.target_seq_len = 5 dec = p.Instantiate() if src_seq_len is None: src_seq_len = 5 src_enc = tf.constant(np.random.uniform(size=(src_seq_len, 2, 8)), tf.float32) if src_enc_padding is None: src_enc_padding = tf.constant( [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 1.0], [1.0, 1.0]], dtype=tf.float32) encoder_outputs = py_utils.NestedMap(encoded=src_enc, padding=src_enc_padding) done_hyps = dec.BeamSearchDecode(encoder_outputs).done_hyps tf.global_variables_initializer().run() softmax_wts = sess.run(dec.vars.softmax) print('softmax wts = ', softmax_wts) done_hyps_serialized = sess.run([done_hyps])[0] hyp = Hypothesis() print('done hyps shape = ', done_hyps_serialized.shape) for i in range(5): for j in range(8): print(i, j, len(done_hyps_serialized[i, j])) hyp.ParseFromString(done_hyps_serialized[2, 5]) print('hyp = ', hyp) return hyp
def _testDecoderBeamSearchDecodeHelperWithOutput(self, params, src_seq_len=None, src_enc_padding=None): config = tf.config_pb2.ConfigProto(graph_options=tf.GraphOptions( optimizer_options=tf.OptimizerOptions(do_function_inlining=False))) p = params with self.session(use_gpu=False, config=config), self.SetEval(True): tf.random.set_seed(837274904) np.random.seed(837575) p.beam_search.num_hyps_per_beam = 4 p.beam_search.length_normalization = 10. p.dtype = tf.float32 p.target_seq_len = 5 dec = p.Instantiate() if src_seq_len is None: src_seq_len = 5 src_enc = tf.constant(np.random.uniform(size=(src_seq_len, 2, 8)), tf.float32) if src_enc_padding is None: src_enc_padding = tf.constant( [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 1.0], [1.0, 1.0]], dtype=tf.float32) encoder_outputs = py_utils.NestedMap(encoded=src_enc, padding=src_enc_padding) topk_hyps = dec.BeamSearchDecode(encoder_outputs).topk_hyps self.evaluate(tf.global_variables_initializer()) softmax_wts = self.evaluate(dec.vars.softmax) print('softmax wts = ', softmax_wts) topk_hyps_serialized = self.evaluate([topk_hyps])[0] hyp = Hypothesis() hyp.ParseFromString(topk_hyps_serialized[1, 2]) return hyp
def testDecoderBeamSearchDecode(self): np.random.seed(837575) p = _DecoderParams(vn_config=py_utils.VariationalNoiseParams( None, False, False), num_classes=8) p.beam_search.num_hyps_per_beam = 4 p.dtype = tf.float32 p.target_seq_len = 5 expected_str = """ beam_id: 1 ids: 0 ids: 6 ids: 2 scores: -2.021608 scores: -2.000098 scores: -2.036338 normalized_score: -0.0550976 atten_vecs { prob: 0.330158 prob: 0.342596 prob: 0.327246 prob: 0.0 prob: 0.0 } atten_vecs { prob: 0.330158 prob: 0.342597 prob: 0.327245 prob: 0.0 prob: 0.0 } atten_vecs { prob: 0.330158 prob: 0.342597 prob: 0.327245 prob: 0.0 prob: 0.0 } """ expected_hyp = Hypothesis() text_format.Parse(expected_str, expected_hyp) decoded_hyp = self._testDecoderBeamSearchDecodeHelperWithOutput( params=p) self._VerifyHypothesesMatch(expected_hyp, decoded_hyp)
def testDecoderBeamSearchDecode(self): np.random.seed(837575) p = self._DecoderParams(vn_config=py_utils.VariationalNoiseParams( None, False, False), num_classes=8) p.beam_search.num_hyps_per_beam = 4 p.dtype = tf.float32 p.target_seq_len = 5 expected_str = """ beam_id: 1 ids: 4 ids: 6 ids: 2 scores: -1.99632573128 scores: -1.99929893017 scores: -1.95849049091 atten_vecs { prob: 0.332584857941 prob: 0.333580821753 prob: 0.333834320307 prob: 0.0 prob: 0.0 } atten_vecs { prob: 0.33258497715 prob: 0.333580613136 prob: 0.333834379911 prob: 0.0 prob: 0.0 } atten_vecs { prob: 0.332585155964 prob: 0.333580434322 prob: 0.333834439516 prob: 0.0 prob: 0.0 } """ expected_hyp = Hypothesis() text_format.Merge(expected_str, expected_hyp) decoded_hyp = self._testDecoderBeamSearchDecodeHelperWithOutput( params=p) self._VerifyHypothesesMatch(expected_hyp, decoded_hyp)
def testBeamSearchDecode(self, dtype=tf.float32): tf.set_random_seed(_TF_RANDOM_SEED) src_batch = 4 src_time = 5 p = self._DecoderParams(dtype=dtype) p.beam_search.num_hyps_per_beam = 2 p.beam_search.coverage_penalty = 0.0 p.beam_search.length_normalization = 0 dec = decoder.TransformerDecoder(p) encoder_outputs, _, _ = self._Inputs(dtype=dtype) decode = dec.BeamSearchDecode(encoder_outputs) # topk_decoded is None in MT decoder, set it to a fake tensor to pass # sess.run(decode). decode = decode._replace(topk_decoded=tf.constant(0, tf.float32)) with self.session(use_gpu=True) as sess: tf.global_variables_initializer().run() actual_decode = sess.run(decode) self.assertTupleEqual( (src_time, src_batch * p.beam_search.num_hyps_per_beam), actual_decode.done_hyps.shape) self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam), actual_decode.topk_hyps.shape) self.assertTupleEqual( (src_batch * p.beam_search.num_hyps_per_beam, src_time), actual_decode.topk_ids.shape) self.assertTupleEqual((src_batch * p.beam_search.num_hyps_per_beam, ), actual_decode.topk_lens.shape) self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam), actual_decode.topk_scores.shape) expected_topk_ids = [[2, 0, 0, 0, 0], [6, 2, 0, 0, 0], [2, 0, 0, 0, 0], [14, 2, 0, 0, 0], [6, 2, 0, 0, 0], [6, 6, 2, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]] expected_topk_lens = [1, 2, 1, 2, 2, 3, 0, 0] expected_topk_scores = [[-2.226787, -4.215915], [-2.232645, -4.228243], [-5.217594, -7.671792], [0., 0.]] # Assert expected IDs etc self.assertAllEqual(expected_topk_ids, actual_decode.topk_ids) self.assertAllEqual(expected_topk_lens, actual_decode.topk_lens) self.assertAllClose(expected_topk_scores, actual_decode.topk_scores) # Assert expected attention probs. hypstr = actual_decode.topk_hyps.flatten()[1] hyp = Hypothesis() hyp.ParseFromString(hypstr) print('HYP:', hyp) atten_vec_0 = list( np.expand_dims(np.array(hyp.atten_vecs[0].prob), 0)[0]) atten_vec_1 = list( np.expand_dims(np.array(hyp.atten_vecs[1].prob), 0)[0]) expected_atten_vec_0 = [0.273083, 0.337312, 0.202556, 0.187049, 0.0] expected_atten_vec_1 = [ 0.19762064516544342, 0.32778304815292358, 0.24845050275325775, 0.22614581882953644, 0.0 ] self.assertAllClose(atten_vec_0, expected_atten_vec_0) self.assertAllClose(atten_vec_1, expected_atten_vec_1) # Test normalized scores of hypotheses. self.assertAlmostEqual(hyp.normalized_score, -4.21591472626, places=4)
def testBeamSearchDecode(self, dtype=tf.float32): tf.set_random_seed(_TF_RANDOM_SEED) src_batch = 4 src_time = 5 p = self._DecoderParams(dtype=dtype) p.beam_search.num_hyps_per_beam = 2 p.beam_search.coverage_penalty = 0.0 p.beam_search.length_normalization = 0 dec = decoder.TransformerDecoder(p) encoder_outputs, _, _ = self._Inputs(dtype=dtype) decode = dec.BeamSearchDecode(encoder_outputs) # topk_decoded is None in MT decoder, set it to a fake tensor to pass # sess.run(decode). decode = decode._replace(topk_decoded=tf.constant(0, tf.float32)) with self.session(use_gpu=True) as sess: tf.global_variables_initializer().run() actual_decode = sess.run(decode) self.assertTupleEqual( (src_time, src_batch * p.beam_search.num_hyps_per_beam), actual_decode.done_hyps.shape) self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam), actual_decode.topk_hyps.shape) self.assertTupleEqual( (src_batch * p.beam_search.num_hyps_per_beam, src_time), actual_decode.topk_ids.shape) self.assertTupleEqual((src_batch * p.beam_search.num_hyps_per_beam, ), actual_decode.topk_lens.shape) self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam), actual_decode.topk_scores.shape) expected_topk_ids = [[2, 0, 0, 0, 0], [6, 2, 0, 0, 0], [14, 2, 0, 0, 0], [14, 14, 2, 0, 0], [2, 0, 0, 0, 0], [19, 2, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]] expected_topk_lens = [1, 2, 2, 3, 1, 2, 0, 0] expected_topk_scores = [[-2.059117, -4.157316], [-4.449532, -6.390632], [-2.486378, -4.982234], [0., 0.]] # Assert expected IDs etc self.assertAllEqual(expected_topk_ids, actual_decode.topk_ids) self.assertAllEqual(expected_topk_lens, actual_decode.topk_lens) self.assertAllClose(expected_topk_scores, actual_decode.topk_scores) # Assert expected attention probs. hypstr = actual_decode.topk_hyps.flatten()[1] hyp = Hypothesis() hyp.ParseFromString(hypstr) print('HYP:', hyp) atten_vec_0 = list( np.expand_dims(np.array(hyp.atten_vecs[0].prob), 0)[0]) atten_vec_1 = list( np.expand_dims(np.array(hyp.atten_vecs[1].prob), 0)[0]) expected_atten_vec_0 = [0.221406, 0.346385, 0.22003, 0.212177, 0.] expected_atten_vec_1 = [0.216729, 0.332198, 0.23248, 0.218592, 0.] self.assertAllClose(atten_vec_0, expected_atten_vec_0) self.assertAllClose(atten_vec_1, expected_atten_vec_1) # Test normalized scores of hypotheses. CompareToGoldenSingleFloat(self, -4.157315, hyp.normalized_score)
def _testBeamSearch(self, expected_values, dtype=tf.float32, init_step_ids=False, has_task_ids=False): tf.set_random_seed(_TF_RANDOM_SEED) src_batch = 4 src_time = 5 p = self._DecoderParams(dtype=dtype, init_step_ids=init_step_ids) p.beam_search.num_hyps_per_beam = 2 p.beam_search.coverage_penalty = 0.0 p.beam_search.length_normalization = 0 dec = decoder.TransformerDecoder(p) encoder_outputs, _, _ = self._Inputs(dtype=dtype, has_task_ids=has_task_ids, init_step_ids=init_step_ids) decode = dec.BeamSearchDecode(encoder_outputs) # topk_decoded is None in MT decoder, set it to a fake tensor to pass # sess.run(decode). decode = decode._replace(topk_decoded=tf.constant(0, tf.float32)) with self.session(use_gpu=True) as sess: tf.global_variables_initializer().run() actual_decode = sess.run(decode) self.assertTupleEqual( (src_time, src_batch * p.beam_search.num_hyps_per_beam), actual_decode.done_hyps.shape) self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam), actual_decode.topk_hyps.shape) self.assertTupleEqual( (src_batch * p.beam_search.num_hyps_per_beam, src_time), actual_decode.topk_ids.shape) self.assertTupleEqual((src_batch * p.beam_search.num_hyps_per_beam, ), actual_decode.topk_lens.shape) self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam), actual_decode.topk_scores.shape) # Assert expected IDs etc self.assertAllEqual(expected_values['topk_ids'], actual_decode.topk_ids) self.assertAllEqual(expected_values['topk_lens'], actual_decode.topk_lens) self.assertAllClose(expected_values['topk_scores'], actual_decode.topk_scores) # Assert expected attention probs. hypstr = actual_decode.topk_hyps.flatten()[1] hyp = Hypothesis() hyp.ParseFromString(hypstr) print('HYP:', hyp) atten_vec_0 = list( np.expand_dims(np.array(hyp.atten_vecs[0].prob), 0)[0]) atten_vec_1 = list( np.expand_dims(np.array(hyp.atten_vecs[1].prob), 0)[0]) self.assertAllClose(atten_vec_0, expected_values['atten_vec_0']) self.assertAllClose(atten_vec_1, expected_values['atten_vec_1']) # Test normalized scores of hypotheses. CompareToGoldenSingleFloat(self, expected_values['normalized_score'], hyp.normalized_score)