def _SameHyp(self, expected_hyp_str, real_serialized_hyp):
hyp1 = hyps_pb2.Hypothesis()
text_format.Parse(expected_hyp_str, hyp1)
hyp2 = hyps_pb2.Hypothesis()
hyp2.ParseFromString(real_serialized_hyp)
self.assertEqual(hyp1.beam_id, hyp2.beam_id)
self.assertEqual(hyp1.ids, hyp2.ids)
self.assertNear(hyp1.normalized_score, hyp2.normalized_score, 1e-6)
self.assertAllClose(hyp1.scores, hyp2.scores)
self.assertEqual(len(hyp1.atten_vecs), len(hyp2.atten_vecs))
for av1, av2 in zip(hyp1.atten_vecs, hyp2.atten_vecs):
self.assertAllClose(av1.prob, av2.prob)
def _SameHyp(self, hyp1_pb, hyp2_pb):
hyp1 = hyps_pb2.Hypothesis()
hyp1.ParseFromString(hyp1_pb)
hyp2 = hyps_pb2.Hypothesis()
hyp2.ParseFromString(hyp2_pb)
self.assertEqual(hyp1.beam_id, hyp2.beam_id)
self.assertEqual(hyp1.ids, hyp2.ids)
self.assertNear(hyp1.normalized_score, hyp2.normalized_score, 1e-6)
self.assertAllClose(hyp1.scores, hyp2.scores)
self.assertEqual(len(hyp1.atten_vecs), len(hyp2.atten_vecs))
for av1, av2 in zip(hyp1.atten_vecs, hyp2.atten_vecs):
self.assertAllClose(av1.prob, av2.prob)
def testBeamSearchOpV2ThreeSteps(self, independence):
"""Similar setup as test_three_steps_eos above but for op V2."""
hyp_size = 2
num_beams = 1
seq_len = 4
small_prob = 1e-7
probs = [
np.log([[0.6, 0.4, small_prob], [0.6, 0.4, small_prob]]),
np.log([[0.55, 0.45, small_prob], [0.05, 0.95, small_prob]]),
# We insert id=1 here to make the decoded output with length 4.
np.log([[small_prob, 1.0, small_prob],
[small_prob, 1.0, small_prob]]),
np.log([[0.05, 0.05, 0.9], [0.05, 0.05, 0.9]]),
]
results = self._runBeamSearchOpHelper(
hyp_size,
num_beams,
seq_len,
_MIN_SCORE,
probs,
independence=independence,
init_atten_probs=tf.zeros([hyp_size, 0]),
atten_probs=np.zeros([seq_len, hyp_size, 0]))
done_hyps = results[-4]
hyp = hyps_pb2.Hypothesis()
hyp.ParseFromString(done_hyps[3, 0])
self.assertAllEqual(0, hyp.beam_id)
self.assertAllEqual([1, 1, 1, 2], hyp.ids)
# [log(0.4), log(0.95), log(1), log(0.9)]
self.assertAllClose([-0.91629070, -0.05129331, 0., -0.10536052],
hyp.scores)
hyp.ParseFromString(done_hyps[3, 1])
self.assertAllEqual(0, hyp.beam_id)
self.assertAllEqual([0, 0, 1, 2], hyp.ids)
# [log(0.6), log(0.55), log(1), log(0.9)]
self.assertAllClose([-0.51082557, -0.59783697, 0., -0.10536052],
hyp.scores)
def testHypsFromBeamSearchOut(self):
with self.session(use_gpu=False) as sess:
hyp_size = 4
num_beams = 2
num_hyps_per_beam = hyp_size / num_beams
src_seq_len = 2
tgt_seq_len = 3
hyps = [
[3, 4, 5, 6],
[7, 8, 9, 10],
[0, 0, 0, 0], # unused row
]
prev_hyps = [
[4, 4, 4, 4], # unused row
[2, 3, 0, 1],
[4, 4, 4, 4], # unused row
]
done_hyps = tf.ones([tgt_seq_len, hyp_size], dtype=tf.bool)
scores = tf.zeros([tgt_seq_len, hyp_size], dtype=tf.float32)
eos_scores = tf.zeros([tgt_seq_len, hyp_size], dtype=tf.float32)
# hyp ids: 0, 1 2 3
# step=0: [[ 0., 1.], [ 2., 3.], [ 4., 5.], [ 6., 7.]]
# step=1: [[ 8., 9.], [10., 11.], [12., 13.], [14., 15.]]
atten_probs = np.reshape(
np.arange(tgt_seq_len * hyp_size * src_seq_len,
dtype=np.float32),
[tgt_seq_len, hyp_size, src_seq_len])
eos_atten_probs = tf.ones([tgt_seq_len, hyp_size, src_seq_len],
dtype=tf.float32)
final_done_hyps = ops.hyps_from_beam_search_outs(
hyps,
prev_hyps,
done_hyps,
scores,
atten_probs,
eos_scores,
eos_atten_probs,
eos_id=2,
num_hyps_per_beam=num_hyps_per_beam,
fix_hyp_atten_vecs=True)
final_done_hyps = sess.run(final_done_hyps)
self.assertAllEqual(final_done_hyps.shape, [tgt_seq_len, hyp_size])
# Focusing on hyps of length 3:
# hyp 0: last step EOS, step 1 hyp 0, step 0 hyp 2 (prev_hyps[1, 0]=2).
# The output seq is [5, 7, 2].
# hyp 1: step 1 hyp 1, step 0 hyp 3, outputs [6, 8, 2].
# hyp 2: step 1 hyp 2, step 0 hyp 0, outputs [3, 9, 2].
# hyp 3: step 1 hyp 3, step 0 hyp 1, outputs [4, 10, 2].
expected_hyp_ids = [[5, 7, 2], [6, 8, 2], [3, 9, 2], [4, 10, 2]]
expected_atten_probs = np.array(
[
[[4, 5], [8, 9], [1, 1]],
[[6, 7], [10, 11], [1, 1]],
[[0, 1], [12, 13], [1, 1]],
[[2, 3], [14, 15], [1, 1]],
],
dtype=np.float32,
)
for hyp_id in range(hyp_size):
hyp = hyps_pb2.Hypothesis()
hyp.ParseFromString(final_done_hyps[2, hyp_id])
self.assertAllEqual(hyp.ids, expected_hyp_ids[hyp_id])
for j in range(tgt_seq_len):
self.assertAllClose(hyp.atten_vecs[j].prob,
expected_atten_probs[hyp_id][j])
def testBeamSearchOpV2Independence(self, independence):
"""Test for V2 op's beam independence mode.
The setup is the following: we have two beams and hyp_per_beam=2.
Beam 0 has the same probablity setup as test_three_steps_eos above,
except that we add a step by inserting id=1 at t=2 so that it finishes
decoding in 4 steps, to [1, 1, 1, 2] (best) and [0, 0, 1, 2] (second best).
Beam 1 encounters a terminated hyp at t=1: [1, 2]. But it also contains
longer terminated hyps at t=3: [1,1,1,2] and [0, 0, 1, 2].
We verify that under beam independence mode, for beam 1 the longer
terminated hyps are not present. We achieve this by setting beam_size to
be very small for force beam_done to True for beam 1.
Args:
independence: whether beam independence mode is enabled.
"""
hyp_size = 4
num_beams = 2
seq_len = 4
small_prob = 1e-7
probs = [
np.log([[0.6, 0.4, small_prob], [0.2, 0.8, small_prob],
[0.6, 0.4, small_prob], [0.2, 0.8, small_prob]]),
np.log([[0.55, 0.45, small_prob], [small_prob, 0.3, 0.6],
[0.05, 0.95, small_prob], [0.05, 0.9, 0.05]]),
# We insert id=1 here to make the decoded output with length 4.
np.log([[small_prob, 1.0,
small_prob], [small_prob, 1.0, small_prob],
[small_prob, 1.0, small_prob],
[small_prob, 1.0, small_prob]]),
np.log([[0.05, 0.05, 0.9], [0.05, 0.05, 0.9], [0.05, 0.05, 0.9],
[0.05, 0.05, 0.9]]),
]
results = self._runBeamSearchOpHelper(
hyp_size,
num_beams,
seq_len,
_MIN_SCORE,
probs,
init_atten_probs=tf.zeros([hyp_size, 0]),
atten_probs=np.zeros([seq_len, hyp_size, 0]),
beam_size=0.1,
independence=independence)
done_hyps = results[-4]
self.assertAllEqual(done_hyps.shape, [4, 4])
hyp = hyps_pb2.Hypothesis()
hyp.ParseFromString(done_hyps[1, 1])
self.assertAllEqual(1, hyp.beam_id)
self.assertAllEqual([1, 2], hyp.ids)
# [log(0.8), log(0.6)]
self.assertAllClose([-0.223144, -0.510826], hyp.scores)
if not independence:
# For beam 1, we have 3 terminated hyps when not under beam independence
# mode.
hyp.ParseFromString(done_hyps[3, 1])
self.assertAllEqual(1, hyp.beam_id)
self.assertAllEqual([1, 1, 1, 2], hyp.ids)
# [log(0.8), log(0.3), log(1), log(0.9)]
self.assertAllClose([-0.22314355, -1.20397282, 0., -0.10536052],
hyp.scores)
hyp.ParseFromString(done_hyps[3, 3])
self.assertAllEqual([0, 1, 1, 2], hyp.ids)
self.assertAllEqual(1, hyp.beam_id)
# [log(0.2), log(0.9), log(1), log(0.9)]
self.assertAllClose([-1.609438, -0.105361, 0., -0.105361],
hyp.scores)
else:
# Under beam independence mode, no further terminated hyps are found.
for step_t in [2, 3]:
for hyp_idx in [1, 3]:
hyp.ParseFromString(done_hyps[step_t, hyp_idx])
self.assertEmpty(hyp.ids)
# For beam 0, we have 2 terminated hyps, similar to in test_three_steps_eos.
hyp.ParseFromString(done_hyps[3, 0])
self.assertAllEqual(0, hyp.beam_id)
self.assertAllEqual([1, 1, 1, 2], hyp.ids)
# [log(0.4), log(0.95), log(1), log(0.9)]
self.assertAllClose([-0.91629070, -0.05129331, 0., -0.10536052],
hyp.scores)
hyp.ParseFromString(done_hyps[3, 2])
self.assertAllEqual(0, hyp.beam_id)
self.assertAllEqual([0, 0, 1, 2], hyp.ids)
# [log(0.6), log(0.55), log(1), log(0.9)]
self.assertAllClose([-0.51082557, -0.59783697, 0., -0.10536052],
hyp.scores)
expected_beam_done = np.array([True, True])
self.assertAllEqual(results[-1], expected_beam_done)
for steps in range(1, 4):
# We verify that beam_done[1] is True after 2 steps (but it has no affect
# when indpendence=False).
results_at_steps = self._runBeamSearchOpHelper(
hyp_size,
num_beams,
seq_len,
_MIN_SCORE,
probs[:steps],
init_atten_probs=tf.zeros([hyp_size, 0]),
atten_probs=np.zeros([seq_len, hyp_size, 0]),
beam_size=0.1,
independence=False)
expected_beam_done = np.array([False, steps >= 2])
self.assertAllEqual(results_at_steps[-1], expected_beam_done)
