def _DecoderFPropHelper(self,
decoder_cls,
dtype,
feed_att_context_to_softmax,
per_example_tensors=False):
with self.session(use_gpu=True):
tf.set_random_seed(_TF_RANDOM_SEED)
p = self._DecoderParams(dtype=dtype, decoder_cls=decoder_cls)
p.per_example_tensors = per_example_tensors
p.feed_attention_context_vec_to_softmax = feed_att_context_to_softmax
dec = p.Instantiate()
encoder_outputs, targets = self._Inputs(dtype=dtype)
fprop_out = dec.FPropDefaultTheta(encoder_outputs, targets)
loss = fprop_out.metrics['loss'][0]
tf.global_variables_initializer().run()
actual_loss = loss.eval()
print('actual loss = ', actual_loss)
if p.feed_attention_context_vec_to_softmax:
CompareToGoldenSingleFloat(self, 7.640674, actual_loss)
else:
CompareToGoldenSingleFloat(self, 7.624605, actual_loss)
if per_example_tensors:
per_example = fprop_out.per_sequence
self.assertIn('loss', per_example)
self.assertAllEqual(per_example['loss'].shape.as_list(), [4])
def testMnistLeNet5(self):
g = tf.Graph()
with g.as_default():
tf.set_random_seed(1618)
p = model_registry.GetParams('image.mnist.LeNet5', 'Test')
p.random_seed = 73234288
p.input.ckpt = self.data_path
p.task.params_init = py_utils.WeightInit.Uniform(0.1, seed=73234288)
with cluster_factory.ForTestingWorker(mode='sync', job='trainer_client'):
model = p.cls(p)
model.ConstructFPropBPropGraph()
with self.session(graph=g) as sess:
sess.run(tf.global_variables_initializer())
CompareToGoldenSingleFloat(self, 2.302583, self._runOneStep(model, sess))
CompareToGoldenSingleFloat(self, 2.302405, self._runOneStep(model, sess))
def _DecoderPerWordAvgLossFPropHelper(self,
decoder_cls,
feed_att_context_to_softmax=False):
with self.session(use_gpu=True):
tf.set_random_seed(_TF_RANDOM_SEED)
p = self._DecoderParams(True)
p.feed_attention_context_vec_to_softmax = feed_att_context_to_softmax
dec = decoder_cls(p)
encoder_outputs, targets = self._Inputs()
loss, _ = dec.FPropDefaultTheta(encoder_outputs, targets)['loss']
tf.global_variables_initializer().run()
actual_loss = loss.eval()
print('actual loss = ', actual_loss)
if p.feed_attention_context_vec_to_softmax:
CompareToGoldenSingleFloat(self, 2.7668, actual_loss)
else:
CompareToGoldenSingleFloat(self, 2.772428, actual_loss)
def testMnistV2(self):
g = tf.Graph()
with g.as_default():
tf.random.set_seed(1618)
p = model_registry.GetParams('test.MnistV2', 'Test')
p.random_seed = 73234288
p.input.ckpt = self.data_path
p.task.params_init = py_utils.WeightInit.Uniform(0.1,
seed=73234288)
with cluster_factory.ForTestingWorker(mode='sync',
job='trainer_client'):
model = p.Instantiate()
model.ConstructFPropBPropGraph()
with self.session(graph=g):
self.evaluate(tf.global_variables_initializer())
CompareToGoldenSingleFloat(self, 2.303070, self._runOneStep(model))
CompareToGoldenSingleFloat(self, 2.297364, self._runOneStep(model))
def testStyleLayer(self):
with self.session(use_gpu=False) as sess:
p = layers_with_attention.StyleLayer.Params().Set(
name='style_layer',
input_dim=10,
output_dim=8,
num_styles=16,
random_seed=28384)
tf.set_random_seed(8372749040)
np.random.seed(12345)
sl = p.cls(p)
features = tf.random_normal([2, 10], seed=28384)
latent, atten_probs = sl.FPropDefaultTheta(features)
tf.global_variables_initializer().run()
latent_v, atten_probs_v = sess.run([latent, atten_probs])
CompareToGoldenSingleFloat(self, 0.122573, np.sum(latent_v))
CompareToGoldenSingleFloat(self, 2.0, np.sum(atten_probs_v))
def _DecoderFPropHelper(self, decoder_cls, dtype,
feed_att_context_to_softmax):
with self.session(use_gpu=True):
tf.set_random_seed(_TF_RANDOM_SEED)
p = self._DecoderParams(dtype=dtype, decoder_cls=decoder_cls)
p.feed_attention_context_vec_to_softmax = feed_att_context_to_softmax
dec = p.Instantiate()
encoder_outputs, targets = self._Inputs(dtype=dtype)
loss, _ = dec.FPropDefaultTheta(encoder_outputs, targets)['loss']
tf.global_variables_initializer().run()
actual_loss = loss.eval()
print('actual loss = ', actual_loss)
if p.feed_attention_context_vec_to_softmax:
CompareToGoldenSingleFloat(self, 7.618915, actual_loss)
else:
CompareToGoldenSingleFloat(self, 7.624220, actual_loss)
def testDecoderFPropWithTaskEmb(self, dtype=tf.float32):
with self.session(use_gpu=True):
tf.random.set_seed(_TF_RANDOM_SEED)
p = self._DecoderParams(dtype=dtype, use_task_emb=True)
dec = decoder.TransformerDecoder(p)
encoder_outputs, targets, _ = self._Inputs(dtype=dtype, has_task_ids=True)
loss, _ = dec.FPropDefaultTheta(encoder_outputs, targets).metrics['loss']
tf.global_variables_initializer().run()
actual_loss = loss.eval()
CompareToGoldenSingleFloat(self, 18.374338, actual_loss)
def _testDecoderFPropHelper(self, decoder_cls, dtype,
feed_att_context_to_softmax):
with self.session(use_gpu=False):
tf.set_random_seed(_TF_RANDOM_SEED)
p = self._DecoderParams(dtype=dtype)
p.feed_attention_context_vec_to_softmax = feed_att_context_to_softmax
dec = decoder_cls(p)
src_enc, src_enc_padding, targets = self._testInputs(dtype=dtype)
loss, _ = dec.FPropDefaultTheta(src_enc, src_enc_padding, targets,
None)['loss']
tf.global_variables_initializer().run()
actual_loss = loss.eval()
print('actual loss = ', actual_loss)
if p.feed_attention_context_vec_to_softmax:
CompareToGoldenSingleFloat(self, 7.613735, actual_loss)
else:
CompareToGoldenSingleFloat(self, 7.624220, actual_loss)
def testTransparentDecoderFProp(self, dtype=tf.float32):
with self.session(use_gpu=True):
tf.random.set_seed(_TF_RANDOM_SEED)
p = self._DecoderParams(is_transparent=True, dtype=dtype)
dec = decoder.TransformerDecoder(p)
encoder_outputs, targets, _ = self._testTransparentInputs(dtype=dtype)
loss, _ = dec.FPropDefaultTheta(encoder_outputs, targets).metrics['loss']
tf.global_variables_initializer().run()
actual_loss = loss.eval()
print('actual loss = ', actual_loss)
CompareToGoldenSingleFloat(self, 19.725393, actual_loss)
def testDecoderFPropWithLangDepAtten(self, dtype=tf.float32):
with self.session(use_gpu=True):
tf.random.set_seed(_TF_RANDOM_SEED)
p = self._DecoderParams(dtype=dtype, use_task_emb=True)
# 4 tasks, 2 languages.
p.use_lang_dependent_atten = True
dec = decoder.TransformerDecoder(p)
encoder_outputs, targets, _ = self._Inputs(dtype=dtype, has_task_ids=True)
loss, _ = dec.FPropDefaultTheta(encoder_outputs, targets).metrics['loss']
tf.global_variables_initializer().run()
actual_loss = loss.eval()
CompareToGoldenSingleFloat(self, 16.200066, actual_loss)
def testDecoderFPropFixedAttentionSeed(self, dtype=tf.float64):
with self.session(use_gpu=True):
tf.set_random_seed(_TF_RANDOM_SEED)
p = self._DecoderParams(dtype=dtype)
p.feed_attention_context_vec_to_softmax = False
p.attention.params_init = py_utils.WeightInit.Gaussian(0.1, 12345)
dec = decoder.MTDecoderV1(p)
encoder_outputs, targets = self._Inputs(dtype=dtype)
loss, _ = dec.FPropDefaultTheta(encoder_outputs, targets)['loss']
tf.global_variables_initializer().run()
actual_loss = loss.eval()
print('actual loss = ', actual_loss)
CompareToGoldenSingleFloat(self, 7.624183, actual_loss)
def testBeamSearchDecode(self, dtype=tf.float32):
tf.set_random_seed(_TF_RANDOM_SEED)
src_batch = 4
src_time = 5
p = self._DecoderParams(dtype=dtype)
p.beam_search.num_hyps_per_beam = 2
p.beam_search.coverage_penalty = 0.0
p.beam_search.length_normalization = 0
dec = decoder.TransformerDecoder(p)
encoder_outputs, _, _ = self._Inputs(dtype=dtype)
decode = dec.BeamSearchDecode(encoder_outputs)
# topk_decoded is None in MT decoder, set it to a fake tensor to pass
# sess.run(decode).
decode = decode._replace(topk_decoded=tf.constant(0, tf.float32))
with self.session(use_gpu=True) as sess:
tf.global_variables_initializer().run()
actual_decode = sess.run(decode)
self.assertTupleEqual(
(src_time, src_batch * p.beam_search.num_hyps_per_beam),
actual_decode.done_hyps.shape)
self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam),
actual_decode.topk_hyps.shape)
self.assertTupleEqual(
(src_batch * p.beam_search.num_hyps_per_beam, src_time),
actual_decode.topk_ids.shape)
self.assertTupleEqual((src_batch * p.beam_search.num_hyps_per_beam, ),
actual_decode.topk_lens.shape)
self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam),
actual_decode.topk_scores.shape)
expected_topk_ids = [[2, 0, 0, 0, 0], [6, 2, 0, 0,
0], [14, 2, 0, 0, 0],
[14, 14, 2, 0, 0], [2, 0, 0, 0, 0],
[19, 2, 0, 0, 0], [0, 0, 0, 0, 0],
[0, 0, 0, 0, 0]]
expected_topk_lens = [1, 2, 2, 3, 1, 2, 0, 0]
expected_topk_scores = [[-2.059117, -4.157316], [-4.449532, -6.390632],
[-2.486378, -4.982234], [0., 0.]]
# Assert expected IDs etc
self.assertAllEqual(expected_topk_ids, actual_decode.topk_ids)
self.assertAllEqual(expected_topk_lens, actual_decode.topk_lens)
self.assertAllClose(expected_topk_scores, actual_decode.topk_scores)
# Assert expected attention probs.
hypstr = actual_decode.topk_hyps.flatten()[1]
hyp = Hypothesis()
hyp.ParseFromString(hypstr)
print('HYP:', hyp)
atten_vec_0 = list(
np.expand_dims(np.array(hyp.atten_vecs[0].prob), 0)[0])
atten_vec_1 = list(
np.expand_dims(np.array(hyp.atten_vecs[1].prob), 0)[0])
expected_atten_vec_0 = [0.221406, 0.346385, 0.22003, 0.212177, 0.]
expected_atten_vec_1 = [0.216729, 0.332198, 0.23248, 0.218592, 0.]
self.assertAllClose(atten_vec_0, expected_atten_vec_0)
self.assertAllClose(atten_vec_1, expected_atten_vec_1)
# Test normalized scores of hypotheses.
CompareToGoldenSingleFloat(self, -4.157315, hyp.normalized_score)
def _testBeamSearch(self,
expected_values,
dtype=tf.float32,
init_step_ids=False,
has_task_ids=False):
tf.set_random_seed(_TF_RANDOM_SEED)
src_batch = 4
src_time = 5
p = self._DecoderParams(dtype=dtype, init_step_ids=init_step_ids)
p.beam_search.num_hyps_per_beam = 2
p.beam_search.coverage_penalty = 0.0
p.beam_search.length_normalization = 0
dec = decoder.TransformerDecoder(p)
encoder_outputs, _, _ = self._Inputs(dtype=dtype,
has_task_ids=has_task_ids,
init_step_ids=init_step_ids)
decode = dec.BeamSearchDecode(encoder_outputs)
# topk_decoded is None in MT decoder, set it to a fake tensor to pass
# sess.run(decode).
decode = decode._replace(topk_decoded=tf.constant(0, tf.float32))
with self.session(use_gpu=True) as sess:
tf.global_variables_initializer().run()
actual_decode = sess.run(decode)
self.assertTupleEqual(
(src_time, src_batch * p.beam_search.num_hyps_per_beam),
actual_decode.done_hyps.shape)
self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam),
actual_decode.topk_hyps.shape)
self.assertTupleEqual(
(src_batch * p.beam_search.num_hyps_per_beam, src_time),
actual_decode.topk_ids.shape)
self.assertTupleEqual((src_batch * p.beam_search.num_hyps_per_beam, ),
actual_decode.topk_lens.shape)
self.assertTupleEqual((src_batch, p.beam_search.num_hyps_per_beam),
actual_decode.topk_scores.shape)
# Assert expected IDs etc
self.assertAllEqual(expected_values['topk_ids'],
actual_decode.topk_ids)
self.assertAllEqual(expected_values['topk_lens'],
actual_decode.topk_lens)
self.assertAllClose(expected_values['topk_scores'],
actual_decode.topk_scores)
# Assert expected attention probs.
hypstr = actual_decode.topk_hyps.flatten()[1]
hyp = Hypothesis()
hyp.ParseFromString(hypstr)
print('HYP:', hyp)
atten_vec_0 = list(
np.expand_dims(np.array(hyp.atten_vecs[0].prob), 0)[0])
atten_vec_1 = list(
np.expand_dims(np.array(hyp.atten_vecs[1].prob), 0)[0])
self.assertAllClose(atten_vec_0, expected_values['atten_vec_0'])
self.assertAllClose(atten_vec_1, expected_values['atten_vec_1'])
# Test normalized scores of hypotheses.
CompareToGoldenSingleFloat(self, expected_values['normalized_score'],
hyp.normalized_score)