def test_rrelu(self, dtype): x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=dtype) lower = 0.1 upper = 0.2 training_results = { np.float16: [-0.288330078, -0.124206543, 0, 1, 2], np.float32: [-0.26851666, -0.116421416, 0, 1, 2], np.float64: [-0.3481333923206531, -0.17150176242558851, 0, 1, 2], } for training in [True, False]: with self.subTest(training=training): tf.random.set_seed(SEED) result = rrelu(x, lower, upper, training=training, seed=SEED) if training: expect_result = training_results.get(dtype) else: expect_result = [ -0.30000001192092896, -0.15000000596046448, 0, 1, 2, ] self.assertAllCloseAccordingToType(result, expect_result)
def benchmarkRreluOp(self): with tf.compat.v1.Session(config=tf.test.benchmark_config()) as sess: x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=np.float32) lower = 0.1 upper = 0.2 result = rrelu(x, lower, upper, training=True) self.run_op_benchmark(sess, result.op, min_iters=25)
def test_unknown_shape(self): fn = rrelu.get_concrete_function( tf.TensorSpec(shape=None, dtype=tf.float32)) for shape in [(1, ), (1, 2), (1, 2, 3), (1, 2, 3, 4)]: x = tf.ones(shape=shape, dtype=tf.float32) self.assertAllClose(fn(x), rrelu(x))
def test_rrelu(self, dtype): x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=dtype) lower = 0.1 upper = 0.2 result = rrelu(x, lower, upper, training=False) expect_result = _ref_rrelu(x, lower, upper) self.assertAllCloseAccordingToType(result, expect_result)
def test_theoretical_gradients(self, dtype): # Only test theoretical gradients for float32 and float64 # because of the instability of float16 while computing jacobian x = tf.constant([-1.5, -0.5, 0.5, 1.5], dtype=dtype) theoretical, numerical = tf.test.compute_gradient( lambda x: rrelu(x), [x]) self.assertAllCloseAccordingToType(theoretical, numerical, atol=1e-4)
def test_gradients(self, dtype): x = tf.constant([-1.5, -0.5, 0.5, 1.5], dtype=dtype) with tf.GradientTape(persistent=True) as tape: tape.watch(x) y_ref = _ref_rrelu(x) y = rrelu(x) grad_ref = tape.gradient(y_ref, x) grad = tape.gradient(y, x) self.assertAllCloseAccordingToType(grad, grad_ref)
def test_theoretical_gradients(dtype, training): x = tf.constant([-2.0, -1.0, -0.1, 0.1, 1.0, 2.0], dtype=dtype) lower = 0.1 upper = 0.2 theoretical, numerical = tf.test.compute_gradient( lambda x: rrelu( x, lower, upper, training=training, seed=None, rng=tf.random.Generator.from_seed(SEED), ), [x], ) test_utils.assert_allclose_according_to_type(theoretical, numerical, rtol=5e-4, atol=5e-4)
def test_theoretical_gradients(self, dtype): x = tf.constant([-2.0, -1.0, -0.1, 0.1, 1.0, 2.0], dtype=dtype) lower = 0.1 upper = 0.2 for training in [True, False]: with self.subTest(training=training): theoretical, numerical = tf.test.compute_gradient( lambda x: rrelu( x, lower, upper, training=training, seed=111111), [x]) # TODO: investigate the difference between CPU and GPU if training is True and tf.test.is_gpu_available() is False: numerical = [[[0.134971, 0., 0., 0., 0., 0.], [0., 0.15648358, 0., 0., 0., 0.], [0., 0., 0.18776372, 0., 0., 0.], [0., 0., 0., 1., 0., 0.], [0., 0., 0., 0., 1., 0.], [0., 0., 0., 0., 0., 1.]]] self.assertAllCloseAccordingToType(theoretical, numerical, rtol=5e-4, atol=5e-4)
def test_rrelu(dtype, training): x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=dtype) lower = 0.1 upper = 0.2 training_results = { np.float16: [-0.3826, -0.165, 0, 1, 2], np.float32: [-0.282151192, -0.199812651, 0, 1, 2], np.float64: [-0.25720977, -0.1221586, 0, 1, 2], } result = rrelu( x, lower, upper, training=training, seed=None, rng=tf.random.Generator.from_seed(SEED), ) if training: expect_result = training_results.get(dtype) else: expect_result = [-0.30000001192092896, -0.15000000596046448, 0, 1, 2] test_utils.assert_allclose_according_to_type(result, expect_result)
def test_rrelu_old(dtype, training): x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=dtype) lower = 0.1 upper = 0.2 tf.random.set_seed(SEED) training_results = { np.float16: [-0.288330078, -0.124206543, 0, 1, 2], np.float32: [-0.26851666, -0.116421416, 0, 1, 2], np.float64: [-0.3481333923206531, -0.17150176242558851, 0, 1, 2], } result = rrelu(x, lower, upper, training=training, seed=SEED) if training: expect_result = training_results.get(dtype) else: expect_result = [ -0.30000001192092896, -0.15000000596046448, 0, 1, 2, ] test_utils.assert_allclose_according_to_type(result, expect_result)
def inner(x): tf.random.set_seed(SEED) return rrelu(x, lower, upper, training=training, seed=SEED)
def test_hardshrink(self, dtype): x = (np.random.rand(2, 3, 4) * 2.0 - 1.0).astype(dtype) self.assertAllCloseAccordingToType(rrelu(x), _ref_rrelu(x)) self.assertAllCloseAccordingToType(rrelu(x, 0, 1), _ref_rrelu(x, 0, 1))
def test_invalid(self): with self.assertRaisesOpError( "lower must be less than or equal to upper."): # pylint: disable=bad-continuation y = rrelu(tf.ones(shape=(1, 2, 3)), lower=0, upper=1) self.evaluate(y)