def test_rrelu(self, dtype):
x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=dtype)
lower = 0.1
upper = 0.2
training_results = {
np.float16: [-0.288330078, -0.124206543, 0, 1, 2],
np.float32: [-0.26851666, -0.116421416, 0, 1, 2],
np.float64: [-0.3481333923206531, -0.17150176242558851, 0, 1, 2],
}
for training in [True, False]:
with self.subTest(training=training):
tf.random.set_seed(SEED)
result = rrelu(x, lower, upper, training=training, seed=SEED)
if training:
expect_result = training_results.get(dtype)
else:
expect_result = [
-0.30000001192092896,
-0.15000000596046448,
0,
1,
2,
]
self.assertAllCloseAccordingToType(result, expect_result)
def benchmarkRreluOp(self):
with tf.compat.v1.Session(config=tf.test.benchmark_config()) as sess:
x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=np.float32)
lower = 0.1
upper = 0.2
result = rrelu(x, lower, upper, training=True)
self.run_op_benchmark(sess, result.op, min_iters=25)
def test_unknown_shape(self):
fn = rrelu.get_concrete_function(
tf.TensorSpec(shape=None, dtype=tf.float32))
for shape in [(1, ), (1, 2), (1, 2, 3), (1, 2, 3, 4)]:
x = tf.ones(shape=shape, dtype=tf.float32)
self.assertAllClose(fn(x), rrelu(x))
def test_rrelu(self, dtype):
x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=dtype)
lower = 0.1
upper = 0.2
result = rrelu(x, lower, upper, training=False)
expect_result = _ref_rrelu(x, lower, upper)
self.assertAllCloseAccordingToType(result, expect_result)
def test_theoretical_gradients(self, dtype):
# Only test theoretical gradients for float32 and float64
# because of the instability of float16 while computing jacobian
x = tf.constant([-1.5, -0.5, 0.5, 1.5], dtype=dtype)
theoretical, numerical = tf.test.compute_gradient(
lambda x: rrelu(x), [x])
self.assertAllCloseAccordingToType(theoretical, numerical, atol=1e-4)
def test_gradients(self, dtype):
x = tf.constant([-1.5, -0.5, 0.5, 1.5], dtype=dtype)
with tf.GradientTape(persistent=True) as tape:
tape.watch(x)
y_ref = _ref_rrelu(x)
y = rrelu(x)
grad_ref = tape.gradient(y_ref, x)
grad = tape.gradient(y, x)
self.assertAllCloseAccordingToType(grad, grad_ref)
def test_theoretical_gradients(dtype, training):
x = tf.constant([-2.0, -1.0, -0.1, 0.1, 1.0, 2.0], dtype=dtype)
lower = 0.1
upper = 0.2
theoretical, numerical = tf.test.compute_gradient(
lambda x: rrelu(
x,
lower,
upper,
training=training,
seed=None,
rng=tf.random.Generator.from_seed(SEED),
),
[x],
)
test_utils.assert_allclose_according_to_type(theoretical,
numerical,
rtol=5e-4,
atol=5e-4)
def test_theoretical_gradients(self, dtype):
x = tf.constant([-2.0, -1.0, -0.1, 0.1, 1.0, 2.0], dtype=dtype)
lower = 0.1
upper = 0.2
for training in [True, False]:
with self.subTest(training=training):
theoretical, numerical = tf.test.compute_gradient(
lambda x: rrelu(
x, lower, upper, training=training, seed=111111), [x])
# TODO: investigate the difference between CPU and GPU
if training is True and tf.test.is_gpu_available() is False:
numerical = [[[0.134971, 0., 0., 0., 0., 0.],
[0., 0.15648358, 0., 0., 0., 0.],
[0., 0., 0.18776372, 0., 0., 0.],
[0., 0., 0., 1., 0., 0.],
[0., 0., 0., 0., 1., 0.],
[0., 0., 0., 0., 0., 1.]]]
self.assertAllCloseAccordingToType(theoretical,
numerical,
rtol=5e-4,
atol=5e-4)
def test_rrelu(dtype, training):
x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=dtype)
lower = 0.1
upper = 0.2
training_results = {
np.float16: [-0.3826, -0.165, 0, 1, 2],
np.float32: [-0.282151192, -0.199812651, 0, 1, 2],
np.float64: [-0.25720977, -0.1221586, 0, 1, 2],
}
result = rrelu(
x,
lower,
upper,
training=training,
seed=None,
rng=tf.random.Generator.from_seed(SEED),
)
if training:
expect_result = training_results.get(dtype)
else:
expect_result = [-0.30000001192092896, -0.15000000596046448, 0, 1, 2]
test_utils.assert_allclose_according_to_type(result, expect_result)
def test_rrelu_old(dtype, training):
x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=dtype)
lower = 0.1
upper = 0.2
tf.random.set_seed(SEED)
training_results = {
np.float16: [-0.288330078, -0.124206543, 0, 1, 2],
np.float32: [-0.26851666, -0.116421416, 0, 1, 2],
np.float64: [-0.3481333923206531, -0.17150176242558851, 0, 1, 2],
}
result = rrelu(x, lower, upper, training=training, seed=SEED)
if training:
expect_result = training_results.get(dtype)
else:
expect_result = [
-0.30000001192092896,
-0.15000000596046448,
0,
1,
2,
]
test_utils.assert_allclose_according_to_type(result, expect_result)
def inner(x):
tf.random.set_seed(SEED)
return rrelu(x, lower, upper, training=training, seed=SEED)
def test_hardshrink(self, dtype):
x = (np.random.rand(2, 3, 4) * 2.0 - 1.0).astype(dtype)
self.assertAllCloseAccordingToType(rrelu(x), _ref_rrelu(x))
self.assertAllCloseAccordingToType(rrelu(x, 0, 1), _ref_rrelu(x, 0, 1))
def test_invalid(self):
with self.assertRaisesOpError(
"lower must be less than or equal to upper."): # pylint: disable=bad-continuation
y = rrelu(tf.ones(shape=(1, 2, 3)), lower=0, upper=1)
self.evaluate(y)
