def res_unit(x, scope_name, rng, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> FixedPointQuantize -> Relu
with nn.parameter_scope("conv1"):
h = PF.fixed_point_quantized_convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.fixed_point_quantize(h)
h = F.relu(h)
# Conv -> BN -> FixedPointQuantize -> Relu
with nn.parameter_scope("conv2"):
h = PF.fixed_point_quantized_convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.fixed_point_quantize(h)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.fixed_point_quantized_convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> FixedPointQuantize -> Relu
h = F.fixed_point_quantize(h)
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def fpq_relu_lenet(image,
test=False,
n=8,
delta=2e-4,
name="fixed-point-relu-graph-ref"):
with nn.parameter_scope(name):
h = PF.convolution(image,
16, (5, 5), (1, 1),
with_bias=False,
name='conv1')
h = PF.batch_normalization(h, batch_stat=not test, name='conv1-bn')
h = F.max_pooling(h, (2, 2))
h = F.fixed_point_quantize(h, n=n, delta=delta, sign=False)
h = PF.convolution(h, 16, (5, 5), (1, 1), with_bias=True, name='conv2')
h = PF.batch_normalization(h, batch_stat=not test, name='conv2-bn')
h = F.max_pooling(h, (2, 2))
h = F.fixed_point_quantize(h, n=n, delta=delta, sign=False)
h = PF.affine(h, 10, with_bias=False, name='fc1')
h = PF.batch_normalization(h, batch_stat=not test, name='fc1-bn')
h = F.fixed_point_quantize(h, n=n, delta=delta, sign=False)
pred = PF.affine(h, 10, with_bias=True, name='fc2')
return pred
def test_fixed_point_quantize_forward_backward(seed,
sign, n, delta, quantize,
ste_fine_grained,
ctx, func_name):
from nbla_test_utils import cap_ignore_region, \
function_tester
rng = np.random.RandomState(seed)
inputs = [
cap_ignore_region(
rng.randn(2, 3, 4).astype(np.float32) * 2,
(-1e-3, 1e-3))]
if quantize:
func_args = [sign, n, delta, quantize, ste_fine_grained]
function_tester(rng,
F.fixed_point_quantize,
ref_fixed_point_quantize,
inputs,
func_args=func_args,
atol_b=1e-3, backward=[True],
ctx=ctx, func_name=func_name,
ref_grad=ref_grad_fixed_point_quantize)
else: # No quantize
for i in inputs:
v = nn.Variable(i.shape)
v.d = i
v.g = np.random.randn(*i.shape)
o = F.fixed_point_quantize(v)
o.forward()
o.backward()
np.allclose(v.d, o.d)
np.allclose(v.g, o.g)
def fpq_resblock(x,
maps,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
test=False,
sign=True,
n=8,
delta=2e-4,
name="convblock"):
h = x
with nn.parameter_scope(name):
h = PF.fixed_point_quantized_convolution(h,
maps,
kernel=kernel,
pad=pad,
stride=stride,
with_bias=False,
sign_w=sign,
n_w=n,
delta_w=delta,
sign_b=sign,
n_b=n,
delta_b=delta)
h = PF.batch_normalization(h, batch_stat=not test)
return F.fixed_point_quantize(h + x, n=n, sign=sign, delta=delta)
def fpq_relu_small_resnet(image,
test=False,
sign=False,
n=8,
delta=2e-4,
name="fixed-point-graph-ref"):
with nn.parameter_scope(name):
h = image
h /= 255.0
h = PF.convolution(h,
16,
kernel=(3, 3),
pad=(1, 1),
name="first-conv",
with_bias=False)
h = PF.batch_normalization(h, name="first-bn", batch_stat=not test)
h = F.fixed_point_quantize(h, sign=sign, n=n, delta=delta)
h = fpq_relu_resblock(h,
maps=16,
test=test,
sign=sign,
n=n,
delta=delta,
name="cb1")
h = fpq_relu_resblock(h,
maps=16,
test=test,
sign=sign,
n=n,
delta=delta,
name="cb2")
h = fpq_relu_resblock(h,
maps=16,
test=test,
sign=sign,
n=n,
delta=delta,
name="cb3")
h = fpq_relu_resblock(h,
maps=16,
test=test,
sign=sign,
n=n,
delta=delta,
name="cb4")
pred = PF.affine(h, 10, name='fc')
return pred
def _fixed_point_activation_conversion(self, activation_func):
# Input
x = activation_func.inputs[0]
x = self.input_map[x] if x in self.input_map else x
# Conversion
n = self.args_fpq["n"]
sign = self.args_fpq["sign"]
delta = self.args_fpq["delta"]
o = F.fixed_point_quantize(x, n=n, sign=sign, delta=delta)
# Map output of ref graph to output of new graph
x = activation_func.outputs[0]
self.input_map[x] = o
# Store output (just in case)
self.outputs.append(o)
return o
def nonl(x, cfg, inplace=False):
# for convenience, store size of x (this allows us to compute the number of activations)
_s = get_parameter_or_create('Asize', (),
ConstantInitializer(np.prod(x.shape[1:])),
need_grad=False)
# get stepsize/maximum value
delta = cfg.a_stepsize
xmax = delta * (2.**cfg.a_bitwidth - 1)
if cfg.a_quantize is not None and 'pow2' in cfg.a_quantize:
xmax = 2.**np.round(np.log2(xmax))
xmin = xmax / 2.**(2.**(cfg.a_bitwidth - 1) - 1)
xmin = np.clip(xmin, cfg.a_xmin_min + 1e-5, cfg.a_xmin_max - 1e-5)
xmax = np.clip(xmax, cfg.a_xmax_min + 1e-5, cfg.a_xmax_max - 1e-5)
print(f'We use default delta ({delta, xmax}) for quantized nonlinearity.')
if cfg.a_quantize == 'fp_relu':
return F.fixed_point_quantize(x,
sign=False,
n=cfg.a_bitwidth,
delta=cfg.a_stepsize)
elif cfg.a_quantize == 'parametric_fp_b_xmax_relu':
return PQ.parametric_fixed_point_quantize_b_xmax(
x,
sign=False,
n_init=cfg.a_bitwidth,
n_min=cfg.a_bitwidth_min,
n_max=cfg.a_bitwidth_max,
xmax_init=xmax,
xmax_min=cfg.a_xmax_min,
xmax_max=cfg.a_xmax_max,
name='Aquant')
elif cfg.a_quantize == 'parametric_fp_d_xmax_relu':
return PQ.parametric_fixed_point_quantize_d_xmax(
x,
sign=False,
d_init=delta,
d_min=cfg.a_stepsize_min,
d_max=cfg.a_stepsize_max,
xmax_init=xmax,
xmax_min=cfg.a_xmax_min,
xmax_max=cfg.a_xmax_max,
name='Aquant')
elif cfg.a_quantize == 'parametric_fp_d_b_relu':
return PQ.parametric_fixed_point_quantize_d_b(x,
sign=False,
n_init=cfg.a_bitwidth,
n_min=cfg.a_bitwidth_min,
n_max=cfg.a_bitwidth_max,
d_init=delta,
d_min=cfg.a_stepsize_min,
d_max=cfg.a_stepsize_max,
name='Aquant')
elif cfg.a_quantize == 'pow2_relu':
return F.pow2_quantize(x,
sign=False,
with_zero=True,
n=cfg.a_bitwidth,
m=np.round(np.log2(xmax)))
elif cfg.a_quantize == 'parametric_pow2_b_xmax_relu':
return PQ.parametric_pow2_quantize_b_xmax(x,
sign=False,
with_zero=True,
n_init=cfg.a_bitwidth,
n_min=cfg.a_bitwidth_min,
n_max=cfg.a_bitwidth_max,
xmax_init=xmax,
xmax_min=cfg.a_xmax_min,
xmax_max=cfg.a_xmax_max,
name='Aquant')
elif cfg.a_quantize == 'parametric_pow2_b_xmin_relu':
return PQ.parametric_pow2_quantize_b_xmin(x,
sign=False,
with_zero=True,
n_init=cfg.a_bitwidth,
n_min=cfg.a_bitwidth_min,
n_max=cfg.a_bitwidth_max,
xmin_init=xmin,
xmin_min=cfg.a_xmin_min,
xmin_max=cfg.a_xmax_max,
name='Aquant')
elif cfg.a_quantize == 'parametric_pow2_xmin_xmax_relu':
return PQ.parametric_pow2_quantize_xmin_xmax(x,
sign=False,
with_zero=True,
xmin_init=xmin,
xmin_min=cfg.a_xmin_min,
xmin_max=cfg.a_xmax_max,
xmax_init=xmax,
xmax_min=cfg.a_xmax_min,
xmax_max=cfg.a_xmax_max,
name='Aquant')
else:
return F.relu(x, inplace=inplace)
def cifar10_fp_net_resnet23_prediction(image, maps=64,
test=False):
"""
Construct Fixed-Point Net using resnet23. Fixed-Point Net quantizes weights
and activations using FixedPointQuantize function.
"""
# Residual Unit
def res_unit(x, scope_name, rng, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> FixedPointQuantize -> Relu
with nn.parameter_scope("conv1"):
h = PF.fixed_point_quantized_convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.fixed_point_quantize(h)
h = F.relu(h)
# Conv -> BN -> FixedPointQuantize -> Relu
with nn.parameter_scope("conv2"):
h = PF.fixed_point_quantized_convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.fixed_point_quantize(h)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.fixed_point_quantized_convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> FixedPointQuantize -> Relu
h = F.fixed_point_quantize(h)
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
ncls = 10
# Conv -> BN -> FixedPointQuantize
with nn.parameter_scope("conv1"):
# Preprocess
image /= 255.0
if not test:
image = F.image_augmentation(image, contrast=1.0,
angle=0.25,
flip_lr=True)
image.need_grad = False
h = PF.fixed_point_quantized_convolution(image, maps, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.fixed_point_quantize(h)
h = res_unit(h, "conv2", False) # -> 32x32
h = res_unit(h, "conv3", True) # -> 16x16
h = res_unit(h, "conv4", False) # -> 16x16
h = res_unit(h, "conv5", True) # -> 8x8
h = res_unit(h, "conv6", False) # -> 8x8
h = res_unit(h, "conv7", True) # -> 4x4
h = res_unit(h, "conv8", False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.fixed_point_quantized_affine(h, ncls)
return pred
