def verify_funcs_are_equivalent(self, dtype): x_np = np.random.uniform(-10, 10, size=(4, 4)).astype(dtype) x = tf.convert_to_tensor(x_np) for approximate in [True, False]: with tf.GradientTape(persistent=True) as t: t.watch(x) y_native = gelu(x, approximate=approximate) y_py = _gelu_py(x, approximate=approximate) self.assertAllCloseAccordingToType(y_native, y_py, atol=1e-4) grad_native = t.gradient(y_native, x) grad_py = t.gradient(y_py, x) self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-4)
def verify_funcs_are_equivalent(dtype, approximate): x_np = np.random.uniform(-10, 10, size=(4, 4)).astype(dtype) x = tf.convert_to_tensor(x_np) with tf.GradientTape(persistent=True) as t: t.watch(x) y_native = gelu(x, approximate=approximate) y_py = _gelu_py(x, approximate=approximate) test_utils.assert_allclose_according_to_type(y_native, y_py) grad_native = t.gradient(y_native, x) grad_py = t.gradient(y_py, x) # TODO: lower atol to 1e-6 # currently it doesn't work. # It necessitates changing the Python or C++ implementation. test_utils.assert_allclose_according_to_type(grad_native, grad_py, atol=1e-5)