def test_stochastic_ternary(): np.random.seed(42) x = np.random.uniform(-0.01, 0.01, size=10) x = np.sort(x) s = stochastic_ternary(alpha="auto_po2", temperature=8) ty = np.zeros_like(s) ts = 0.0 n = 1000 for _ in range(n): y = K.eval(s(K.constant(x))) scale = K.eval(s.scale)[0] ts = ts + scale ty = ty + (y / scale) result = (ty / n).astype(np.float32) scale = np.array([ts / n]) expected = np.array([ -0.998, -0.992, -0.992, -0.208, 0.048, 0.04, 0.448, 0.606, 0.987, 0.998 ]).astype(np.float32) expected_scale = np.array([0.007812]) assert_allclose(result, expected, atol=0.1) assert_allclose(scale, expected_scale, rtol=0.1)
def test_stochastic_ternary_inference_mode(alpha, threshold, test_values, expected_values): K.set_learning_phase(0) x = K.placeholder(ndim=2) q = stochastic_ternary(alpha, threshold) f = K.function([x], [q(x)]) result = f([test_values])[0] assert_allclose(result, expected_values, rtol=1e-05)
def test_stochastic_ternary(bound, alpha, temperature, expected_values, expected_scale): np.random.seed(42) K.set_learning_phase(1) n = 1000 x = np.random.uniform(-bound, bound, size=(n, 10)) x = np.sort(x, axis=1) s = stochastic_ternary(alpha=alpha, temperature=temperature) y = K.eval(s(K.constant(x))) scale = K.eval(s.scale).astype(np.float32)[0] ty = np.zeros_like(s) for i in range(n): ty = ty + (y[i] / scale) result = (ty/n).astype(np.float32) assert_allclose(result, expected_values, atol=0.1) assert_allclose(scale, expected_scale, rtol=0.1)
def main(): # check the mean value of samples from stochastic_rounding for po2 np.random.seed(42) count = 100000 val = 42 a = K.constant([val] * count) b = quantized_po2(use_stochastic_rounding=True)(a) res = np.sum(K.eval(b)) / count print(res, "should be close to ", val) b = quantized_relu_po2(use_stochastic_rounding=True)(a) res = np.sum(K.eval(b)) / count print(res, "should be close to ", val) a = K.constant([-1] * count) b = quantized_relu_po2(use_stochastic_rounding=True)(a) res = np.sum(K.eval(b)) / count print(res, "should be all ", 0) # non-stochastic rounding quantizer. a = K.constant([-3.0, -2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0, 3.0]) a = K.constant([0.194336]) print(" a =", K.eval(a).astype(np.float16)) print("qa =", K.eval(quantized_relu(6,2)(a)).astype(np.float16)) print("ss =", K.eval(smooth_sigmoid(a)).astype(np.float16)) print("hs =", K.eval(hard_sigmoid(a)).astype(np.float16)) print("ht =", K.eval(hard_tanh(a)).astype(np.float16)) print("st =", K.eval(smooth_tanh(a)).astype(np.float16)) c = K.constant(np.arange(-1.5, 1.51, 0.3)) print(" c =", K.eval(c).astype(np.float16)) print("qb_111 =", K.eval(quantized_bits(1,1,1)(c)).astype(np.float16)) print("qb_210 =", K.eval(quantized_bits(2,1,0)(c)).astype(np.float16)) print("qb_211 =", K.eval(quantized_bits(2,1,1)(c)).astype(np.float16)) print("qb_300 =", K.eval(quantized_bits(3,0,0)(c)).astype(np.float16)) print("qb_301 =", K.eval(quantized_bits(3,0,1)(c)).astype(np.float16)) c_1000 = K.constant(np.array([list(K.eval(c))] * 1000)) b = np.sum(K.eval(bernoulli()(c_1000)).astype(np.int32), axis=0) / 1000.0 print(" hs =", K.eval(hard_sigmoid(c)).astype(np.float16)) print(" b_all =", b.astype(np.float16)) T = 0.0 t = K.eval(stochastic_ternary(alpha="auto")(c_1000)) for i in range(10): print("stochastic_ternary({}) =".format(i), t[i]) print(" st_all =", np.round( np.sum(t.astype(np.float32), axis=0).astype(np.float16) / 1000.0, 2).astype(np.float16)) print(" ternary =", K.eval(ternary(threshold=0.5)(c)).astype(np.int32)) c = K.constant(np.arange(-1.5, 1.51, 0.3)) print(" c =", K.eval(c).astype(np.float16)) print(" b_10 =", K.eval(binary(1)(c)).astype(np.float16)) print("qr_10 =", K.eval(quantized_relu(1,0)(c)).astype(np.float16)) print("qr_11 =", K.eval(quantized_relu(1,1)(c)).astype(np.float16)) print("qr_20 =", K.eval(quantized_relu(2,0)(c)).astype(np.float16)) print("qr_21 =", K.eval(quantized_relu(2,1)(c)).astype(np.float16)) print("qr_101 =", K.eval(quantized_relu(1,0,1)(c)).astype(np.float16)) print("qr_111 =", K.eval(quantized_relu(1,1,1)(c)).astype(np.float16)) print("qr_201 =", K.eval(quantized_relu(2,0,1)(c)).astype(np.float16)) print("qr_211 =", K.eval(quantized_relu(2,1,1)(c)).astype(np.float16)) print("qt_200 =", K.eval(quantized_tanh(2,0)(c)).astype(np.float16)) print("qt_210 =", K.eval(quantized_tanh(2,1)(c)).astype(np.float16)) print("qt_201 =", K.eval(quantized_tanh(2,0,1)(c)).astype(np.float16)) print("qt_211 =", K.eval(quantized_tanh(2,1,1)(c)).astype(np.float16)) set_internal_sigmoid("smooth"); print("with smooth sigmoid") print("qr_101 =", K.eval(quantized_relu(1,0,1)(c)).astype(np.float16)) print("qr_111 =", K.eval(quantized_relu(1,1,1)(c)).astype(np.float16)) print("qr_201 =", K.eval(quantized_relu(2,0,1)(c)).astype(np.float16)) print("qr_211 =", K.eval(quantized_relu(2,1,1)(c)).astype(np.float16)) print("qt_200 =", K.eval(quantized_tanh(2,0)(c)).astype(np.float16)) print("qt_210 =", K.eval(quantized_tanh(2,1)(c)).astype(np.float16)) print("qt_201 =", K.eval(quantized_tanh(2,0,1)(c)).astype(np.float16)) print("qt_211 =", K.eval(quantized_tanh(2,1,1)(c)).astype(np.float16)) set_internal_sigmoid("real"); print("with real sigmoid") print("qr_101 =", K.eval(quantized_relu(1,0,1)(c)).astype(np.float16)) print("qr_111 =", K.eval(quantized_relu(1,1,1)(c)).astype(np.float16)) print("qr_201 =", K.eval(quantized_relu(2,0,1)(c)).astype(np.float16)) print("qr_211 =", K.eval(quantized_relu(2,1,1)(c)).astype(np.float16)) print("qt_200 =", K.eval(quantized_tanh(2,0)(c)).astype(np.float16)) print("qt_210 =", K.eval(quantized_tanh(2,1)(c)).astype(np.float16)) print("qt_201 =", K.eval(quantized_tanh(2,0,1)(c)).astype(np.float16)) print("qt_211 =", K.eval(quantized_tanh(2,1,1)(c)).astype(np.float16)) set_internal_sigmoid("hard") print(" c =", K.eval(c).astype(np.float16)) print("q2_31 =", K.eval(quantized_po2(3,1)(c)).astype(np.float16)) print("q2_32 =", K.eval(quantized_po2(3,2)(c)).astype(np.float16)) print("qr2_21 =", K.eval(quantized_relu_po2(2,1)(c)).astype(np.float16)) print("qr2_22 =", K.eval(quantized_relu_po2(2,2)(c)).astype(np.float16)) print("qr2_44 =", K.eval(quantized_relu_po2(4,1)(c)).astype(np.float16)) # stochastic rounding c = K.constant(np.arange(-1.5, 1.51, 0.3)) print("q2_32_2 =", K.eval(quantized_relu_po2(32,2)(c)).astype(np.float16)) b = K.eval(stochastic_binary()(c_1000)).astype(np.int32) for i in range(5): print("sbinary({}) =".format(i), b[i]) print("sbinary =", np.round(np.sum(b, axis=0) / 1000.0, 2).astype(np.float16)) print(" binary =", K.eval(binary()(c)).astype(np.int32)) print(" c =", K.eval(c).astype(np.float16)) for i in range(10): print(" s_bin({}) =".format(i), K.eval(binary(use_stochastic_rounding=1)(c)).astype(np.int32)) for i in range(10): print(" s_po2({}) =".format(i), K.eval(quantized_po2(use_stochastic_rounding=1)(c)).astype(np.int32)) for i in range(10): print( " s_relu_po2({}) =".format(i), K.eval(quantized_relu_po2(use_stochastic_rounding=1)(c)).astype( np.int32))
def main(): np.random.seed(42) a = K.constant([-3.0, -2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0, 3.0]) a = K.constant([0.194336]) print(" a =", K.eval(a).astype(np.float16)) print("qa =", K.eval(quantized_relu(6,2)(a)).astype(np.float16)) print("ss =", K.eval(smooth_sigmoid(a)).astype(np.float16)) print("hs =", K.eval(hard_sigmoid(a)).astype(np.float16)) print("ht =", K.eval(hard_tanh(a)).astype(np.float16)) print("st =", K.eval(smooth_tanh(a)).astype(np.float16)) c = K.constant(np.arange(-1.5, 1.51, 0.3)) print(" c =", K.eval(c).astype(np.float16)) print("qb_111 =", K.eval(quantized_bits(1,1,1)(c)).astype(np.float16)) print("qb_210 =", K.eval(quantized_bits(2,1,0)(c)).astype(np.float16)) print("qb_211 =", K.eval(quantized_bits(2,1,1)(c)).astype(np.float16)) print("qb_300 =", K.eval(quantized_bits(3,0,0)(c)).astype(np.float16)) print("qb_301 =", K.eval(quantized_bits(3,0,1)(c)).astype(np.float16)) c_1000 = K.constant(np.array([list(K.eval(c))] * 1000)) b = np.sum(K.eval(bernoulli()(c_1000)).astype(np.int32), axis=0) / 1000.0 print(" hs =", K.eval(hard_sigmoid(c)).astype(np.float16)) print(" b_all =", b.astype(np.float16)) T = 0.0 t = K.eval(stochastic_ternary(threshold=T)(c_1000)).astype(np.int32) for i in range(10): print("sternary({}) =".format(i), t[i]) print(" st_all =", np.round( np.sum(t.astype(np.float32), axis=0).astype(np.float16) / 1000.0, 2).astype(np.float16)) print(" ternary =", K.eval(ternary(threshold=0.5)(c)).astype(np.int32)) b = K.eval(stochastic_binary()(c_1000)).astype(np.int32) for i in range(5): print("sbinary({}) =".format(i), b[i]) print("sbinary =", np.round(np.sum(b, axis=0) / 1000.0, 2).astype(np.float16)) print(" binary =", K.eval(binary()(c)).astype(np.int32)) c = K.constant(np.arange(-1.5, 1.51, 0.3)) print(" c =", K.eval(c).astype(np.float16)) print(" b_10 =", K.eval(binary(1)(c)).astype(np.float16)) print("qr_10 =", K.eval(quantized_relu(1,0)(c)).astype(np.float16)) print("qr_11 =", K.eval(quantized_relu(1,1)(c)).astype(np.float16)) print("qr_20 =", K.eval(quantized_relu(2,0)(c)).astype(np.float16)) print("qr_21 =", K.eval(quantized_relu(2,1)(c)).astype(np.float16)) print("qr_101 =", K.eval(quantized_relu(1,0,1)(c)).astype(np.float16)) print("qr_111 =", K.eval(quantized_relu(1,1,1)(c)).astype(np.float16)) print("qr_201 =", K.eval(quantized_relu(2,0,1)(c)).astype(np.float16)) print("qr_211 =", K.eval(quantized_relu(2,1,1)(c)).astype(np.float16)) print("qt_200 =", K.eval(quantized_tanh(2,0)(c)).astype(np.float16)) print("qt_210 =", K.eval(quantized_tanh(2,1)(c)).astype(np.float16)) print("qt_201 =", K.eval(quantized_tanh(2,0,1)(c)).astype(np.float16)) print("qt_211 =", K.eval(quantized_tanh(2,1,1)(c)).astype(np.float16)) set_internal_sigmoid("smooth"); print("with smooth sigmoid") print("qr_101 =", K.eval(quantized_relu(1,0,1)(c)).astype(np.float16)) print("qr_111 =", K.eval(quantized_relu(1,1,1)(c)).astype(np.float16)) print("qr_201 =", K.eval(quantized_relu(2,0,1)(c)).astype(np.float16)) print("qr_211 =", K.eval(quantized_relu(2,1,1)(c)).astype(np.float16)) print("qt_200 =", K.eval(quantized_tanh(2,0)(c)).astype(np.float16)) print("qt_210 =", K.eval(quantized_tanh(2,1)(c)).astype(np.float16)) print("qt_201 =", K.eval(quantized_tanh(2,0,1)(c)).astype(np.float16)) print("qt_211 =", K.eval(quantized_tanh(2,1,1)(c)).astype(np.float16)) set_internal_sigmoid("real"); print("with real sigmoid") print("qr_101 =", K.eval(quantized_relu(1,0,1)(c)).astype(np.float16)) print("qr_111 =", K.eval(quantized_relu(1,1,1)(c)).astype(np.float16)) print("qr_201 =", K.eval(quantized_relu(2,0,1)(c)).astype(np.float16)) print("qr_211 =", K.eval(quantized_relu(2,1,1)(c)).astype(np.float16)) print("qt_200 =", K.eval(quantized_tanh(2,0)(c)).astype(np.float16)) print("qt_210 =", K.eval(quantized_tanh(2,1)(c)).astype(np.float16)) print("qt_201 =", K.eval(quantized_tanh(2,0,1)(c)).astype(np.float16)) print("qt_211 =", K.eval(quantized_tanh(2,1,1)(c)).astype(np.float16)) set_internal_sigmoid("hard") print(" c =", K.eval(c).astype(np.float16)) print("q2_31 =", K.eval(quantized_po2(3,1)(c)).astype(np.float16)) print("q2_32 =", K.eval(quantized_po2(3,2)(c)).astype(np.float16)) print("qr2_21 =", K.eval(quantized_relu_po2(2,1)(c)).astype(np.float16)) print("qr2_22 =", K.eval(quantized_relu_po2(2,2)(c)).astype(np.float16)) print("qr2_44 =", K.eval(quantized_relu_po2(4,1)(c)).astype(np.float16)) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") print("q2_32_2 =", K.eval(quantized_relu_po2(32,2)(c)).astype(np.float16)) assert len(w) == 1 assert issubclass(w[-1].category, UserWarning) print(str(w[-1].message))