def test_dorefa_quantize(self, k_bit, mode):
x = tf.keras.backend.placeholder(ndim=2)
f = tf.keras.backend.function([x],
[lq.quantizers.DoReFa(k_bit, mode)(x)])
real_values = testing_utils.generate_real_values_with_zeros()
result = f([real_values])[0]
n = 2**k_bit - 1
if mode == "weights":
# Create the preprocessed and scaled stimulus, which is then ready to
# go through the same test like for the activation quantizer
divider = np.amax(np.abs(np.tanh(real_values)))
real_values = np.tanh(real_values) / divider
real_values = (real_values / 2.0) + 0.5
# The results, which are currently on [-1, 1] range get the same
# scaling, so they behave like they were created on the activation
# range and can be tested like that
result = result / 2.0 + 0.5
assert not np.any(result > 1)
assert not np.any(result < 0)
for i in range(n + 1):
np.testing.assert_allclose(
result[(real_values > (2 * i - 1) / (2 * n))
& (real_values < (2 * i + 1) / (2 * n))],
i / n,
atol=1e-6,
)
def test_identity_ste_grad(self, eager_mode, fn):
x = testing_utils.generate_real_values_with_zeros(shape=(8, 3, 3, 16))
tf_x = tf.Variable(x)
with tf.GradientTape() as tape:
activation = fn(tf_x)
grad = tape.gradient(activation, tf_x)
np.testing.assert_allclose(grad.numpy(), np.ones_like(x))
def test_heaviside(fn):
x = tf.keras.backend.placeholder(ndim=2)
f = tf.keras.backend.function([x], [fn(x)])
binarized_values = np.random.choice([0, 1], size=(2, 5))
result = f([binarized_values])[0]
np.testing.assert_allclose(result, binarized_values)
real_values = generate_real_values_with_zeros()
result = f([real_values])[0]
assert np.all(result[real_values <= 0] == 0)
assert np.all(result[real_values > 0] == 1)
def test_ternarization_with_default_threshold(self, fn):
x = tf.keras.backend.placeholder(ndim=2)
test_threshold = 0.05 # This is the default
f = tf.keras.backend.function([x], [lq.quantizers.get(fn)(x)])
real_values = testing_utils.generate_real_values_with_zeros()
result = f([real_values])[0]
assert np.all(result[real_values > test_threshold] == 1)
assert np.all(result[real_values < -test_threshold] == -1)
assert np.all(result[np.abs(real_values) < test_threshold] == 0)
assert not np.any(result > 1)
assert not np.any(result < -1)
def test_and_binarization(self, fn):
x = tf.keras.backend.placeholder(ndim=2)
f = tf.keras.backend.function([x], [lq.quantizers.get(fn)(x)])
binarized_values = np.random.choice([0, 1], size=(2, 5))
result = f([binarized_values])[0]
np.testing.assert_allclose(result, binarized_values)
real_values = testing_utils.generate_real_values_with_zeros()
result = f([real_values])[0]
assert np.all(result[real_values <= 0] == 0)
assert np.all(result[real_values > 0] == 1)
def test_ste_grad(self, eager_mode, fn):
@np.vectorize
def ste_grad(x):
if np.abs(x) <= 1:
return 1.0
return 0.0
x = testing_utils.generate_real_values_with_zeros(shape=(8, 3, 3, 16))
tf_x = tf.Variable(x)
with tf.GradientTape() as tape:
activation = fn(tf_x)
grad = tape.gradient(activation, tf_x)
np.testing.assert_allclose(grad.numpy(), ste_grad(x))
def test_ternarization_with_custom_threshold(self):
x = tf.keras.backend.placeholder(ndim=2)
test_threshold = np.random.uniform(0.01, 0.8)
fn = lq.quantizers.SteTern(threshold_value=test_threshold)
f = tf.keras.backend.function([x], [fn(x)])
real_values = testing_utils.generate_real_values_with_zeros()
result = f([real_values])[0]
assert np.all(result[real_values > test_threshold] == 1)
assert np.all(result[real_values < -test_threshold] == -1)
assert np.all(result[np.abs(real_values) < test_threshold] == 0)
assert not np.any(result > 1)
assert not np.any(result < -1)
def test_dorefa_quantize(self, fn):
x = tf.keras.backend.placeholder(ndim=2)
f = tf.keras.backend.function([x], [fn(x)])
real_values = testing_utils.generate_real_values_with_zeros()
result = f([real_values])[0]
k_bit = 2
n = 2**k_bit - 1
assert not np.any(result > 1)
assert not np.any(result < 0)
for i in range(n + 1):
assert np.all(result[(real_values > (2 * i - 1) / (2 * n))
& (real_values < (2 * i + 1) /
(2 * n))] == i / n)
def test_approx_sign_grad(self, eager_mode):
@np.vectorize
def approx_sign_grad(x):
if np.abs(x) <= 1:
return 2 - 2 * np.abs(x)
return 0.0
x = testing_utils.generate_real_values_with_zeros(shape=(8, 3, 3, 16))
tf_x = tf.Variable(x)
with tf.GradientTape() as tape:
activation = lq.quantizers.ApproxSign()(tf_x)
grad = tape.gradient(activation, tf_x)
np.testing.assert_allclose(grad.numpy(), approx_sign_grad(x))
def test_dorefa_ste_grad(self, eager_mode):
@np.vectorize
def ste_grad(x):
if x <= 1 and x >= 0:
return 1.0
return 0.0
x = testing_utils.generate_real_values_with_zeros(shape=(8, 3, 3, 16))
tf_x = tf.Variable(x)
with tf.GradientTape() as tape:
activation = lq.quantizers.DoReFaQuantizer(2)(tf_x)
grad = tape.gradient(activation, tf_x)
np.testing.assert_allclose(grad.numpy(), ste_grad(x))
def test_ternarization_with_ternary_weight_networks(self):
x = tf.keras.backend.placeholder(ndim=2)
real_values = testing_utils.generate_real_values_with_zeros()
test_threshold = 0.7 * np.sum(np.abs(real_values)) / np.size(real_values)
fn = lq.quantizers.SteTern(ternary_weight_networks=True)
f = tf.keras.backend.function([x], [fn(x)])
result = f([real_values])[0]
assert np.all(result[real_values > test_threshold] == 1)
assert np.all(result[real_values < -test_threshold] == -1)
assert np.all(result[np.abs(real_values) < test_threshold] == 0)
assert not np.any(result > 1)
assert not np.any(result < -1)
def test_leaky_tanh():
@np.vectorize
def leaky_tanh(x, alpha):
if x <= -1:
return -1 + alpha * (x + 1)
elif x <= 1:
return x
else:
return 1 + alpha * (x - 1)
real_values = generate_real_values_with_zeros()
x = tf.keras.backend.placeholder(ndim=2)
f = tf.keras.backend.function([x], [lq.activations.leaky_tanh(x)])
result = f([real_values])[0]
np.testing.assert_allclose(result, leaky_tanh(real_values, alpha=0.2))
def test_swish_grad(self, eager_mode):
def swish_grad(x, beta):
return (beta * (2 - beta * x * np.tanh(beta * x / 2)) /
(1 + np.cosh(beta * x)))
x = testing_utils.generate_real_values_with_zeros(shape=(8, 3, 3, 16))
tf_x = tf.Variable(x)
with tf.GradientTape() as tape:
activation = lq.quantizers.SwishSign()(tf_x)
grad = tape.gradient(activation, tf_x)
np.testing.assert_allclose(grad.numpy(), swish_grad(x, beta=5.0))
with tf.GradientTape() as tape:
activation = lq.quantizers.SwishSign(beta=10.0)(tf_x)
grad = tape.gradient(activation, tf_x)
np.testing.assert_allclose(grad.numpy(), swish_grad(x, beta=10.0))
def test_sign(fn):
x = tf.keras.backend.placeholder(ndim=2)
f = tf.keras.backend.function([x], [fn(x)])
binarized_values = np.random.choice([-1, 1],
size=(2, 5)).astype(np.float32)
result = f(binarized_values)[0]
np.testing.assert_allclose(result, binarized_values)
real_values = generate_real_values_with_zeros()
result = f(real_values)[0]
assert not np.any(result == 0)
assert np.all(result[real_values < 0] == -1)
assert np.all(result[real_values >= 0] == 1)
zero_values = np.zeros((2, 5))
result = f(zero_values)[0]
assert np.all(result == 1)
def test_ternarization_basic(self, fn):
x = tf.keras.backend.placeholder(ndim=2)
f = tf.keras.backend.function([x], [lq.quantizers.get(fn)(x)])
ternarized_values = np.random.choice([-1, 0, 1], size=(4, 10))
result = f([ternarized_values])[0]
np.testing.assert_allclose(result, ternarized_values)
assert not np.any(result > 1)
assert not np.any(result < -1)
assert np.any(result == -1)
assert np.any(result == 1)
assert np.any(result == 0)
real_values = testing_utils.generate_real_values_with_zeros()
result = f([real_values])[0]
assert not np.any(result > 1)
assert not np.any(result < -1)
assert np.any(result == -1)
assert np.any(result == 1)
assert np.any(result == 0)
def test_dorefa_ste_grad(self, mode):
@np.vectorize
def ste_grad(x):
if x <= 1 and x >= 0:
return 1.0
return 0.0
def tanh_grad(x):
# 1/(cosh**2) is the derivative of tanh. The gradients of the
# scaling operations cancel each other and the gradient of the
# quantizek function is supposed to be 1 everywhere, because it
# is used on its linear region only. tanh does all the limiting.
dividend = np.amax(np.abs(np.tanh(x)))
return 1 / (np.cosh(x)**2.0) / dividend
expected_gradient = ste_grad if mode == "activations" else tanh_grad
x = testing_utils.generate_real_values_with_zeros(shape=(8, 3, 3, 16))
tf_x = tf.Variable(x)
with tf.GradientTape() as tape:
activation = lq.quantizers.DoReFa(2, mode)(tf_x)
grad = tape.gradient(activation, tf_x)
np.testing.assert_allclose(grad.numpy(), expected_gradient(x))
def test_clip():
real_values = generate_real_values_with_zeros()
clip_instance = lq.constraints.weight_clip(clip_value=0.75)
result = clip_instance(tf.keras.backend.variable(real_values))
result = tf.keras.backend.eval(result)
np.testing.assert_allclose(result, np.clip(real_values, -0.75, 0.75))
def test_hard_tanh():
real_values = generate_real_values_with_zeros()
x = tf.keras.backend.placeholder(ndim=2)
f = tf.keras.backend.function([x], [lq.activations.hard_tanh(x)])
result = f([real_values])[0]
np.testing.assert_allclose(result, np.clip(real_values, -1, 1))
