def _SameHyp(self, expected_hyp_str, real_serialized_hyp): hyp1 = hyps_pb2.Hypothesis() text_format.Parse(expected_hyp_str, hyp1) hyp2 = hyps_pb2.Hypothesis() hyp2.ParseFromString(real_serialized_hyp) self.assertEqual(hyp1.beam_id, hyp2.beam_id) self.assertEqual(hyp1.ids, hyp2.ids) self.assertNear(hyp1.normalized_score, hyp2.normalized_score, 1e-6) self.assertAllClose(hyp1.scores, hyp2.scores) self.assertEqual(len(hyp1.atten_vecs), len(hyp2.atten_vecs)) for av1, av2 in zip(hyp1.atten_vecs, hyp2.atten_vecs): self.assertAllClose(av1.prob, av2.prob)
def _SameHyp(self, hyp1_pb, hyp2_pb): hyp1 = hyps_pb2.Hypothesis() hyp1.ParseFromString(hyp1_pb) hyp2 = hyps_pb2.Hypothesis() hyp2.ParseFromString(hyp2_pb) self.assertEqual(hyp1.beam_id, hyp2.beam_id) self.assertEqual(hyp1.ids, hyp2.ids) self.assertNear(hyp1.normalized_score, hyp2.normalized_score, 1e-6) self.assertAllClose(hyp1.scores, hyp2.scores) self.assertEqual(len(hyp1.atten_vecs), len(hyp2.atten_vecs)) for av1, av2 in zip(hyp1.atten_vecs, hyp2.atten_vecs): self.assertAllClose(av1.prob, av2.prob)
def testBeamSearchOpV2ThreeSteps(self, independence): """Similar setup as test_three_steps_eos above but for op V2.""" hyp_size = 2 num_beams = 1 seq_len = 4 small_prob = 1e-7 probs = [ np.log([[0.6, 0.4, small_prob], [0.6, 0.4, small_prob]]), np.log([[0.55, 0.45, small_prob], [0.05, 0.95, small_prob]]), # We insert id=1 here to make the decoded output with length 4. np.log([[small_prob, 1.0, small_prob], [small_prob, 1.0, small_prob]]), np.log([[0.05, 0.05, 0.9], [0.05, 0.05, 0.9]]), ] results = self._runBeamSearchOpHelper( hyp_size, num_beams, seq_len, _MIN_SCORE, probs, independence=independence, init_atten_probs=tf.zeros([hyp_size, 0]), atten_probs=np.zeros([seq_len, hyp_size, 0])) done_hyps = results[-4] hyp = hyps_pb2.Hypothesis() hyp.ParseFromString(done_hyps[3, 0]) self.assertAllEqual(0, hyp.beam_id) self.assertAllEqual([1, 1, 1, 2], hyp.ids) # [log(0.4), log(0.95), log(1), log(0.9)] self.assertAllClose([-0.91629070, -0.05129331, 0., -0.10536052], hyp.scores) hyp.ParseFromString(done_hyps[3, 1]) self.assertAllEqual(0, hyp.beam_id) self.assertAllEqual([0, 0, 1, 2], hyp.ids) # [log(0.6), log(0.55), log(1), log(0.9)] self.assertAllClose([-0.51082557, -0.59783697, 0., -0.10536052], hyp.scores)
def testHypsFromBeamSearchOut(self): with self.session(use_gpu=False) as sess: hyp_size = 4 num_beams = 2 num_hyps_per_beam = hyp_size / num_beams src_seq_len = 2 tgt_seq_len = 3 hyps = [ [3, 4, 5, 6], [7, 8, 9, 10], [0, 0, 0, 0], # unused row ] prev_hyps = [ [4, 4, 4, 4], # unused row [2, 3, 0, 1], [4, 4, 4, 4], # unused row ] done_hyps = tf.ones([tgt_seq_len, hyp_size], dtype=tf.bool) scores = tf.zeros([tgt_seq_len, hyp_size], dtype=tf.float32) eos_scores = tf.zeros([tgt_seq_len, hyp_size], dtype=tf.float32) # hyp ids: 0, 1 2 3 # step=0: [[ 0., 1.], [ 2., 3.], [ 4., 5.], [ 6., 7.]] # step=1: [[ 8., 9.], [10., 11.], [12., 13.], [14., 15.]] atten_probs = np.reshape( np.arange(tgt_seq_len * hyp_size * src_seq_len, dtype=np.float32), [tgt_seq_len, hyp_size, src_seq_len]) eos_atten_probs = tf.ones([tgt_seq_len, hyp_size, src_seq_len], dtype=tf.float32) final_done_hyps = ops.hyps_from_beam_search_outs( hyps, prev_hyps, done_hyps, scores, atten_probs, eos_scores, eos_atten_probs, eos_id=2, num_hyps_per_beam=num_hyps_per_beam, fix_hyp_atten_vecs=True) final_done_hyps = sess.run(final_done_hyps) self.assertAllEqual(final_done_hyps.shape, [tgt_seq_len, hyp_size]) # Focusing on hyps of length 3: # hyp 0: last step EOS, step 1 hyp 0, step 0 hyp 2 (prev_hyps[1, 0]=2). # The output seq is [5, 7, 2]. # hyp 1: step 1 hyp 1, step 0 hyp 3, outputs [6, 8, 2]. # hyp 2: step 1 hyp 2, step 0 hyp 0, outputs [3, 9, 2]. # hyp 3: step 1 hyp 3, step 0 hyp 1, outputs [4, 10, 2]. expected_hyp_ids = [[5, 7, 2], [6, 8, 2], [3, 9, 2], [4, 10, 2]] expected_atten_probs = np.array( [ [[4, 5], [8, 9], [1, 1]], [[6, 7], [10, 11], [1, 1]], [[0, 1], [12, 13], [1, 1]], [[2, 3], [14, 15], [1, 1]], ], dtype=np.float32, ) for hyp_id in range(hyp_size): hyp = hyps_pb2.Hypothesis() hyp.ParseFromString(final_done_hyps[2, hyp_id]) self.assertAllEqual(hyp.ids, expected_hyp_ids[hyp_id]) for j in range(tgt_seq_len): self.assertAllClose(hyp.atten_vecs[j].prob, expected_atten_probs[hyp_id][j])
def testBeamSearchOpV2Independence(self, independence): """Test for V2 op's beam independence mode. The setup is the following: we have two beams and hyp_per_beam=2. Beam 0 has the same probablity setup as test_three_steps_eos above, except that we add a step by inserting id=1 at t=2 so that it finishes decoding in 4 steps, to [1, 1, 1, 2] (best) and [0, 0, 1, 2] (second best). Beam 1 encounters a terminated hyp at t=1: [1, 2]. But it also contains longer terminated hyps at t=3: [1,1,1,2] and [0, 0, 1, 2]. We verify that under beam independence mode, for beam 1 the longer terminated hyps are not present. We achieve this by setting beam_size to be very small for force beam_done to True for beam 1. Args: independence: whether beam independence mode is enabled. """ hyp_size = 4 num_beams = 2 seq_len = 4 small_prob = 1e-7 probs = [ np.log([[0.6, 0.4, small_prob], [0.2, 0.8, small_prob], [0.6, 0.4, small_prob], [0.2, 0.8, small_prob]]), np.log([[0.55, 0.45, small_prob], [small_prob, 0.3, 0.6], [0.05, 0.95, small_prob], [0.05, 0.9, 0.05]]), # We insert id=1 here to make the decoded output with length 4. np.log([[small_prob, 1.0, small_prob], [small_prob, 1.0, small_prob], [small_prob, 1.0, small_prob], [small_prob, 1.0, small_prob]]), np.log([[0.05, 0.05, 0.9], [0.05, 0.05, 0.9], [0.05, 0.05, 0.9], [0.05, 0.05, 0.9]]), ] results = self._runBeamSearchOpHelper( hyp_size, num_beams, seq_len, _MIN_SCORE, probs, init_atten_probs=tf.zeros([hyp_size, 0]), atten_probs=np.zeros([seq_len, hyp_size, 0]), beam_size=0.1, independence=independence) done_hyps = results[-4] self.assertAllEqual(done_hyps.shape, [4, 4]) hyp = hyps_pb2.Hypothesis() hyp.ParseFromString(done_hyps[1, 1]) self.assertAllEqual(1, hyp.beam_id) self.assertAllEqual([1, 2], hyp.ids) # [log(0.8), log(0.6)] self.assertAllClose([-0.223144, -0.510826], hyp.scores) if not independence: # For beam 1, we have 3 terminated hyps when not under beam independence # mode. hyp.ParseFromString(done_hyps[3, 1]) self.assertAllEqual(1, hyp.beam_id) self.assertAllEqual([1, 1, 1, 2], hyp.ids) # [log(0.8), log(0.3), log(1), log(0.9)] self.assertAllClose([-0.22314355, -1.20397282, 0., -0.10536052], hyp.scores) hyp.ParseFromString(done_hyps[3, 3]) self.assertAllEqual([0, 1, 1, 2], hyp.ids) self.assertAllEqual(1, hyp.beam_id) # [log(0.2), log(0.9), log(1), log(0.9)] self.assertAllClose([-1.609438, -0.105361, 0., -0.105361], hyp.scores) else: # Under beam independence mode, no further terminated hyps are found. for step_t in [2, 3]: for hyp_idx in [1, 3]: hyp.ParseFromString(done_hyps[step_t, hyp_idx]) self.assertEmpty(hyp.ids) # For beam 0, we have 2 terminated hyps, similar to in test_three_steps_eos. hyp.ParseFromString(done_hyps[3, 0]) self.assertAllEqual(0, hyp.beam_id) self.assertAllEqual([1, 1, 1, 2], hyp.ids) # [log(0.4), log(0.95), log(1), log(0.9)] self.assertAllClose([-0.91629070, -0.05129331, 0., -0.10536052], hyp.scores) hyp.ParseFromString(done_hyps[3, 2]) self.assertAllEqual(0, hyp.beam_id) self.assertAllEqual([0, 0, 1, 2], hyp.ids) # [log(0.6), log(0.55), log(1), log(0.9)] self.assertAllClose([-0.51082557, -0.59783697, 0., -0.10536052], hyp.scores) expected_beam_done = np.array([True, True]) self.assertAllEqual(results[-1], expected_beam_done) for steps in range(1, 4): # We verify that beam_done[1] is True after 2 steps (but it has no affect # when indpendence=False). results_at_steps = self._runBeamSearchOpHelper( hyp_size, num_beams, seq_len, _MIN_SCORE, probs[:steps], init_atten_probs=tf.zeros([hyp_size, 0]), atten_probs=np.zeros([seq_len, hyp_size, 0]), beam_size=0.1, independence=False) expected_beam_done = np.array([False, steps >= 2]) self.assertAllEqual(results_at_steps[-1], expected_beam_done)