def testScaleGradientsCheckNumerics(self):
"""ScaleGradients when enable_check_numerics=True."""
FLAGS.enable_check_numerics = True
p = self.TestParams()
p.input = base_input_generator.BaseSequenceInputGenerator.Params()
task = p.Instantiate()
task.CreateVariable(
'a',
py_utils.WeightParams(shape=[],
init=py_utils.WeightInit.Constant(0)))
var_a = task.theta.a
# Make a NaN gradient.
var_grads = py_utils.NestedMap(a=(var_a, 0. * tf.log(0.)))
scaled_grads_map = task.learners[0].ScaleGradients(var_grads)
with self.session():
tf.global_variables_initializer().run()
with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
'is not finite'):
self.assertTrue(scaled_grads_map.has_nan_or_inf.eval())
self.assertEqual(0., scaled_grads_map.grad_scale.eval())
# The final gradient must be finite.
self.assertFalse(
tf.is_nan(scaled_grads_map.final_var_grads.a[1]).eval())
self.assertTrue(
tf.is_finite(scaled_grads_map.final_var_grads.a[1]).eval())
def testScaleGradients(self):
p = self.TestParams()
p.input = base_input_generator.BaseSequenceInputGenerator.Params()
task = p.Instantiate()
task.CreateVariable(
'a',
py_utils.WeightParams(shape=[], init=py_utils.WeightInit.Constant(0)))
var_a = task.theta.a
var_grads = py_utils.NestedMap(a=(var_a, tf.ones_like(var_a)))
scaled_grads_map = task.learners[0].ScaleGradients(var_grads)
FLAGS.enable_check_numerics = False
with self.session():
tf.global_variables_initializer().run()
self.assertEqual(1.0, scaled_grads_map.grad_scale.eval())
# The final gradient must be finite.
self.assertFalse(tf.is_nan(scaled_grads_map.final_var_grads.a[1]).eval())
self.assertTrue(
tf.is_finite(scaled_grads_map.final_var_grads.a[1]).eval())
