def verify_funcs_are_equivalent(self, dtype):
x_np = np.random.uniform(-10, 10, size=(4, 4)).astype(dtype)
x = tf.convert_to_tensor(x_np)
for approximate in [True, False]:
with tf.GradientTape(persistent=True) as t:
t.watch(x)
y_native = gelu(x, approximate=approximate)
y_py = _gelu_py(x, approximate=approximate)
self.assertAllCloseAccordingToType(y_native, y_py, atol=1e-4)
grad_native = t.gradient(y_native, x)
grad_py = t.gradient(y_py, x)
self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-4)
def verify_funcs_are_equivalent(dtype, approximate):
x_np = np.random.uniform(-10, 10, size=(4, 4)).astype(dtype)
x = tf.convert_to_tensor(x_np)
with tf.GradientTape(persistent=True) as t:
t.watch(x)
y_native = gelu(x, approximate=approximate)
y_py = _gelu_py(x, approximate=approximate)
test_utils.assert_allclose_according_to_type(y_native, y_py)
grad_native = t.gradient(y_native, x)
grad_py = t.gradient(y_py, x)
# TODO: lower atol to 1e-6
# currently it doesn't work.
# It necessitates changing the Python or C++ implementation.
test_utils.assert_allclose_according_to_type(grad_native, grad_py, atol=1e-5)