def testScaleGradientsCheckNumerics(self): """ScaleGradients when enable_check_numerics=True.""" FLAGS.enable_check_numerics = True p = self.TestParams() p.input = base_input_generator.BaseSequenceInputGenerator.Params() task = p.Instantiate() task.CreateVariable( 'a', py_utils.WeightParams(shape=[], init=py_utils.WeightInit.Constant(0))) var_a = task.theta.a # Make a NaN gradient. var_grads = py_utils.NestedMap(a=(var_a, 0. * tf.log(0.))) scaled_grads_map = task.learners[0].ScaleGradients(var_grads) with self.session(): tf.global_variables_initializer().run() with self.assertRaisesRegexp(tf.errors.InvalidArgumentError, 'is not finite'): self.assertTrue(scaled_grads_map.has_nan_or_inf.eval()) self.assertEqual(0., scaled_grads_map.grad_scale.eval()) # The final gradient must be finite. self.assertFalse( tf.is_nan(scaled_grads_map.final_var_grads.a[1]).eval()) self.assertTrue( tf.is_finite(scaled_grads_map.final_var_grads.a[1]).eval())
def testScaleGradients(self): p = self.TestParams() p.input = base_input_generator.BaseSequenceInputGenerator.Params() task = p.Instantiate() task.CreateVariable( 'a', py_utils.WeightParams(shape=[], init=py_utils.WeightInit.Constant(0))) var_a = task.theta.a var_grads = py_utils.NestedMap(a=(var_a, tf.ones_like(var_a))) scaled_grads_map = task.learners[0].ScaleGradients(var_grads) FLAGS.enable_check_numerics = False with self.session(): tf.global_variables_initializer().run() self.assertEqual(1.0, scaled_grads_map.grad_scale.eval()) # The final gradient must be finite. self.assertFalse(tf.is_nan(scaled_grads_map.final_var_grads.a[1]).eval()) self.assertTrue( tf.is_finite(scaled_grads_map.final_var_grads.a[1]).eval())