def _testDecoderBeamSearchDecodeHelperWithOutput(self,
params,
src_seq_len=None,
src_enc_padding=None):
config = tf.ConfigProto(graph_options=tf.GraphOptions(
optimizer_options=tf.OptimizerOptions(do_function_inlining=False)))
p = params
with self.session(use_gpu=False,
config=config) as sess, self.SetEval(True):
tf.set_random_seed(837274904)
np.random.seed(837575)
p.beam_search.num_hyps_per_beam = 4
p.dtype = tf.float32
p.target_seq_len = 5
dec = p.Instantiate()
if src_seq_len is None:
src_seq_len = 5
src_enc = tf.constant(np.random.uniform(size=(src_seq_len, 2, 8)),
tf.float32)
if src_enc_padding is None:
src_enc_padding = tf.constant(
[[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 1.0],
[1.0, 1.0]],
dtype=tf.float32)
encoder_outputs = py_utils.NestedMap(encoded=src_enc,
padding=src_enc_padding)
done_hyps = dec.BeamSearchDecode(encoder_outputs).done_hyps
tf.global_variables_initializer().run()
softmax_wts = sess.run(dec.vars.softmax)
print('softmax wts = ', softmax_wts)
done_hyps_serialized = sess.run([done_hyps])[0]
hyp = Hypothesis()
print('done hyps shape = ', done_hyps_serialized.shape)
for i in range(5):
for j in range(8):
print(i, j, len(done_hyps_serialized[i, j]))
hyp.ParseFromString(done_hyps_serialized[2, 5])
print('hyp = ', hyp)
return hyp
def _testDecoderBeamSearchDecodeHelperWithOutput(self,
params,
src_seq_len=None,
src_enc_padding=None):
config = tf.config_pb2.ConfigProto(graph_options=tf.GraphOptions(
optimizer_options=tf.OptimizerOptions(do_function_inlining=False)))
p = params
with self.session(use_gpu=False, config=config), self.SetEval(True):
tf.random.set_seed(837274904)
np.random.seed(837575)
p.beam_search.num_hyps_per_beam = 4
p.beam_search.length_normalization = 10.
p.dtype = tf.float32
p.target_seq_len = 5
dec = p.Instantiate()
if src_seq_len is None:
src_seq_len = 5
src_enc = tf.constant(np.random.uniform(size=(src_seq_len, 2, 8)),
tf.float32)
if src_enc_padding is None:
src_enc_padding = tf.constant(
[[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 1.0],
[1.0, 1.0]],
dtype=tf.float32)
encoder_outputs = py_utils.NestedMap(encoded=src_enc,
padding=src_enc_padding)
topk_hyps = dec.BeamSearchDecode(encoder_outputs).topk_hyps
self.evaluate(tf.global_variables_initializer())
softmax_wts = self.evaluate(dec.vars.softmax)
print('softmax wts = ', softmax_wts)
topk_hyps_serialized = self.evaluate([topk_hyps])[0]
hyp = Hypothesis()
hyp.ParseFromString(topk_hyps_serialized[1, 2])
return hyp
def testDecoderBeamSearchDecode(self):
np.random.seed(837575)
p = _DecoderParams(vn_config=py_utils.VariationalNoiseParams(
None, False, False),
num_classes=8)
p.beam_search.num_hyps_per_beam = 4
p.dtype = tf.float32
p.target_seq_len = 5
expected_str = """
beam_id: 1
ids: 0
ids: 6
ids: 2
scores: -2.021608
scores: -2.000098
scores: -2.036338
normalized_score: -0.0550976
atten_vecs {
prob: 0.330158
prob: 0.342596
prob: 0.327246
prob: 0.0
prob: 0.0
}
atten_vecs {
prob: 0.330158
prob: 0.342597
prob: 0.327245
prob: 0.0
prob: 0.0
}
atten_vecs {
prob: 0.330158
prob: 0.342597
prob: 0.327245
prob: 0.0
prob: 0.0
}
"""
expected_hyp = Hypothesis()
text_format.Parse(expected_str, expected_hyp)
decoded_hyp = self._testDecoderBeamSearchDecodeHelperWithOutput(
params=p)
self._VerifyHypothesesMatch(expected_hyp, decoded_hyp)
def testDecoderBeamSearchDecode(self):
np.random.seed(837575)
p = self._DecoderParams(vn_config=py_utils.VariationalNoiseParams(
None, False, False),
num_classes=8)
p.beam_search.num_hyps_per_beam = 4
p.dtype = tf.float32
p.target_seq_len = 5
expected_str = """
beam_id: 1
ids: 4
ids: 6
ids: 2
scores: -1.99632573128
scores: -1.99929893017
scores: -1.95849049091
atten_vecs {
prob: 0.332584857941
prob: 0.333580821753
prob: 0.333834320307
prob: 0.0
prob: 0.0
}
atten_vecs {
prob: 0.33258497715
prob: 0.333580613136
prob: 0.333834379911
prob: 0.0
prob: 0.0
}
atten_vecs {
prob: 0.332585155964
prob: 0.333580434322
prob: 0.333834439516
prob: 0.0
prob: 0.0
}
"""
expected_hyp = Hypothesis()
text_format.Merge(expected_str, expected_hyp)
decoded_hyp = self._testDecoderBeamSearchDecodeHelperWithOutput(
params=p)
self._VerifyHypothesesMatch(expected_hyp, decoded_hyp)
def testBeamSearchDecode(self, dtype=tf.float32):
tf.set_random_seed(_TF_RANDOM_SEED)
src_batch = 4
src_time = 5
p = self._DecoderParams(dtype=dtype)
p.beam_search.num_hyps_per_beam = 2
p.beam_search.coverage_penalty = 0.0
p.beam_search.length_normalization = 0
dec = decoder.TransformerDecoder(p)
encoder_outputs, _, _ = self._Inputs(dtype=dtype)
decode = dec.BeamSearchDecode(encoder_outputs)
# topk_decoded is None in MT decoder, set it to a fake tensor to pass
# sess.run(decode).
decode = decode._replace(topk_decoded=tf.constant(0, tf.float32))
with self.session(use_gpu=True) as sess:
tf.global_variables_initializer().run()
actual_decode = sess.run(decode)
self.assertTupleEqual(
(src_time, src_batch * p.beam_search.num_hyps_per_beam),
actual_decode.done_hyps.shape)
self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam),
actual_decode.topk_hyps.shape)
self.assertTupleEqual(
(src_batch * p.beam_search.num_hyps_per_beam, src_time),
actual_decode.topk_ids.shape)
self.assertTupleEqual((src_batch * p.beam_search.num_hyps_per_beam, ),
actual_decode.topk_lens.shape)
self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam),
actual_decode.topk_scores.shape)
expected_topk_ids = [[2, 0, 0, 0, 0], [6, 2, 0, 0, 0], [2, 0, 0, 0, 0],
[14, 2, 0, 0, 0], [6, 2, 0, 0, 0],
[6, 6, 2, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]
expected_topk_lens = [1, 2, 1, 2, 2, 3, 0, 0]
expected_topk_scores = [[-2.226787, -4.215915], [-2.232645, -4.228243],
[-5.217594, -7.671792], [0., 0.]]
# Assert expected IDs etc
self.assertAllEqual(expected_topk_ids, actual_decode.topk_ids)
self.assertAllEqual(expected_topk_lens, actual_decode.topk_lens)
self.assertAllClose(expected_topk_scores, actual_decode.topk_scores)
# Assert expected attention probs.
hypstr = actual_decode.topk_hyps.flatten()[1]
hyp = Hypothesis()
hyp.ParseFromString(hypstr)
print('HYP:', hyp)
atten_vec_0 = list(
np.expand_dims(np.array(hyp.atten_vecs[0].prob), 0)[0])
atten_vec_1 = list(
np.expand_dims(np.array(hyp.atten_vecs[1].prob), 0)[0])
expected_atten_vec_0 = [0.273083, 0.337312, 0.202556, 0.187049, 0.0]
expected_atten_vec_1 = [
0.19762064516544342, 0.32778304815292358, 0.24845050275325775,
0.22614581882953644, 0.0
]
self.assertAllClose(atten_vec_0, expected_atten_vec_0)
self.assertAllClose(atten_vec_1, expected_atten_vec_1)
# Test normalized scores of hypotheses.
self.assertAlmostEqual(hyp.normalized_score, -4.21591472626, places=4)
def testBeamSearchDecode(self, dtype=tf.float32):
tf.set_random_seed(_TF_RANDOM_SEED)
src_batch = 4
src_time = 5
p = self._DecoderParams(dtype=dtype)
p.beam_search.num_hyps_per_beam = 2
p.beam_search.coverage_penalty = 0.0
p.beam_search.length_normalization = 0
dec = decoder.TransformerDecoder(p)
encoder_outputs, _, _ = self._Inputs(dtype=dtype)
decode = dec.BeamSearchDecode(encoder_outputs)
# topk_decoded is None in MT decoder, set it to a fake tensor to pass
# sess.run(decode).
decode = decode._replace(topk_decoded=tf.constant(0, tf.float32))
with self.session(use_gpu=True) as sess:
tf.global_variables_initializer().run()
actual_decode = sess.run(decode)
self.assertTupleEqual(
(src_time, src_batch * p.beam_search.num_hyps_per_beam),
actual_decode.done_hyps.shape)
self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam),
actual_decode.topk_hyps.shape)
self.assertTupleEqual(
(src_batch * p.beam_search.num_hyps_per_beam, src_time),
actual_decode.topk_ids.shape)
self.assertTupleEqual((src_batch * p.beam_search.num_hyps_per_beam, ),
actual_decode.topk_lens.shape)
self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam),
actual_decode.topk_scores.shape)
expected_topk_ids = [[2, 0, 0, 0, 0], [6, 2, 0, 0,
0], [14, 2, 0, 0, 0],
[14, 14, 2, 0, 0], [2, 0, 0, 0, 0],
[19, 2, 0, 0, 0], [0, 0, 0, 0, 0],
[0, 0, 0, 0, 0]]
expected_topk_lens = [1, 2, 2, 3, 1, 2, 0, 0]
expected_topk_scores = [[-2.059117, -4.157316], [-4.449532, -6.390632],
[-2.486378, -4.982234], [0., 0.]]
# Assert expected IDs etc
self.assertAllEqual(expected_topk_ids, actual_decode.topk_ids)
self.assertAllEqual(expected_topk_lens, actual_decode.topk_lens)
self.assertAllClose(expected_topk_scores, actual_decode.topk_scores)
# Assert expected attention probs.
hypstr = actual_decode.topk_hyps.flatten()[1]
hyp = Hypothesis()
hyp.ParseFromString(hypstr)
print('HYP:', hyp)
atten_vec_0 = list(
np.expand_dims(np.array(hyp.atten_vecs[0].prob), 0)[0])
atten_vec_1 = list(
np.expand_dims(np.array(hyp.atten_vecs[1].prob), 0)[0])
expected_atten_vec_0 = [0.221406, 0.346385, 0.22003, 0.212177, 0.]
expected_atten_vec_1 = [0.216729, 0.332198, 0.23248, 0.218592, 0.]
self.assertAllClose(atten_vec_0, expected_atten_vec_0)
self.assertAllClose(atten_vec_1, expected_atten_vec_1)
# Test normalized scores of hypotheses.
CompareToGoldenSingleFloat(self, -4.157315, hyp.normalized_score)
def _testBeamSearch(self,
expected_values,
dtype=tf.float32,
init_step_ids=False,
has_task_ids=False):
tf.set_random_seed(_TF_RANDOM_SEED)
src_batch = 4
src_time = 5
p = self._DecoderParams(dtype=dtype, init_step_ids=init_step_ids)
p.beam_search.num_hyps_per_beam = 2
p.beam_search.coverage_penalty = 0.0
p.beam_search.length_normalization = 0
dec = decoder.TransformerDecoder(p)
encoder_outputs, _, _ = self._Inputs(dtype=dtype,
has_task_ids=has_task_ids,
init_step_ids=init_step_ids)
decode = dec.BeamSearchDecode(encoder_outputs)
# topk_decoded is None in MT decoder, set it to a fake tensor to pass
# sess.run(decode).
decode = decode._replace(topk_decoded=tf.constant(0, tf.float32))
with self.session(use_gpu=True) as sess:
tf.global_variables_initializer().run()
actual_decode = sess.run(decode)
self.assertTupleEqual(
(src_time, src_batch * p.beam_search.num_hyps_per_beam),
actual_decode.done_hyps.shape)
self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam),
actual_decode.topk_hyps.shape)
self.assertTupleEqual(
(src_batch * p.beam_search.num_hyps_per_beam, src_time),
actual_decode.topk_ids.shape)
self.assertTupleEqual((src_batch * p.beam_search.num_hyps_per_beam, ),
actual_decode.topk_lens.shape)
self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam),
actual_decode.topk_scores.shape)
# Assert expected IDs etc
self.assertAllEqual(expected_values['topk_ids'],
actual_decode.topk_ids)
self.assertAllEqual(expected_values['topk_lens'],
actual_decode.topk_lens)
self.assertAllClose(expected_values['topk_scores'],
actual_decode.topk_scores)
# Assert expected attention probs.
hypstr = actual_decode.topk_hyps.flatten()[1]
hyp = Hypothesis()
hyp.ParseFromString(hypstr)
print('HYP:', hyp)
atten_vec_0 = list(
np.expand_dims(np.array(hyp.atten_vecs[0].prob), 0)[0])
atten_vec_1 = list(
np.expand_dims(np.array(hyp.atten_vecs[1].prob), 0)[0])
self.assertAllClose(atten_vec_0, expected_values['atten_vec_0'])
self.assertAllClose(atten_vec_1, expected_values['atten_vec_1'])
# Test normalized scores of hypotheses.
CompareToGoldenSingleFloat(self, expected_values['normalized_score'],
hyp.normalized_score)
