def res_unit(x, scope_name, rng, dn=False, test=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> FixedPointQuantize -> Relu with nn.parameter_scope("conv1"): h = PF.fixed_point_quantized_convolution(x, C / 2, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.fixed_point_quantize(h) h = F.relu(h) # Conv -> BN -> FixedPointQuantize -> Relu with nn.parameter_scope("conv2"): h = PF.fixed_point_quantized_convolution(h, C / 2, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.fixed_point_quantize(h) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): h = PF.fixed_point_quantized_convolution(h, C, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> FixedPointQuantize -> Relu h = F.fixed_point_quantize(h) h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h
def fpq_relu_lenet(image, test=False, n=8, delta=2e-4, name="fixed-point-relu-graph-ref"): with nn.parameter_scope(name): h = PF.convolution(image, 16, (5, 5), (1, 1), with_bias=False, name='conv1') h = PF.batch_normalization(h, batch_stat=not test, name='conv1-bn') h = F.max_pooling(h, (2, 2)) h = F.fixed_point_quantize(h, n=n, delta=delta, sign=False) h = PF.convolution(h, 16, (5, 5), (1, 1), with_bias=True, name='conv2') h = PF.batch_normalization(h, batch_stat=not test, name='conv2-bn') h = F.max_pooling(h, (2, 2)) h = F.fixed_point_quantize(h, n=n, delta=delta, sign=False) h = PF.affine(h, 10, with_bias=False, name='fc1') h = PF.batch_normalization(h, batch_stat=not test, name='fc1-bn') h = F.fixed_point_quantize(h, n=n, delta=delta, sign=False) pred = PF.affine(h, 10, with_bias=True, name='fc2') return pred
def test_fixed_point_quantize_forward_backward(seed, sign, n, delta, quantize, ste_fine_grained, ctx, func_name): from nbla_test_utils import cap_ignore_region, \ function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3))] if quantize: func_args = [sign, n, delta, quantize, ste_fine_grained] function_tester(rng, F.fixed_point_quantize, ref_fixed_point_quantize, inputs, func_args=func_args, atol_b=1e-3, backward=[True], ctx=ctx, func_name=func_name, ref_grad=ref_grad_fixed_point_quantize) else: # No quantize for i in inputs: v = nn.Variable(i.shape) v.d = i v.g = np.random.randn(*i.shape) o = F.fixed_point_quantize(v) o.forward() o.backward() np.allclose(v.d, o.d) np.allclose(v.g, o.g)
def fpq_resblock(x, maps, kernel=(3, 3), pad=(1, 1), stride=(1, 1), test=False, sign=True, n=8, delta=2e-4, name="convblock"): h = x with nn.parameter_scope(name): h = PF.fixed_point_quantized_convolution(h, maps, kernel=kernel, pad=pad, stride=stride, with_bias=False, sign_w=sign, n_w=n, delta_w=delta, sign_b=sign, n_b=n, delta_b=delta) h = PF.batch_normalization(h, batch_stat=not test) return F.fixed_point_quantize(h + x, n=n, sign=sign, delta=delta)
def fpq_relu_small_resnet(image, test=False, sign=False, n=8, delta=2e-4, name="fixed-point-graph-ref"): with nn.parameter_scope(name): h = image h /= 255.0 h = PF.convolution(h, 16, kernel=(3, 3), pad=(1, 1), name="first-conv", with_bias=False) h = PF.batch_normalization(h, name="first-bn", batch_stat=not test) h = F.fixed_point_quantize(h, sign=sign, n=n, delta=delta) h = fpq_relu_resblock(h, maps=16, test=test, sign=sign, n=n, delta=delta, name="cb1") h = fpq_relu_resblock(h, maps=16, test=test, sign=sign, n=n, delta=delta, name="cb2") h = fpq_relu_resblock(h, maps=16, test=test, sign=sign, n=n, delta=delta, name="cb3") h = fpq_relu_resblock(h, maps=16, test=test, sign=sign, n=n, delta=delta, name="cb4") pred = PF.affine(h, 10, name='fc') return pred
def _fixed_point_activation_conversion(self, activation_func): # Input x = activation_func.inputs[0] x = self.input_map[x] if x in self.input_map else x # Conversion n = self.args_fpq["n"] sign = self.args_fpq["sign"] delta = self.args_fpq["delta"] o = F.fixed_point_quantize(x, n=n, sign=sign, delta=delta) # Map output of ref graph to output of new graph x = activation_func.outputs[0] self.input_map[x] = o # Store output (just in case) self.outputs.append(o) return o
def nonl(x, cfg, inplace=False): # for convenience, store size of x (this allows us to compute the number of activations) _s = get_parameter_or_create('Asize', (), ConstantInitializer(np.prod(x.shape[1:])), need_grad=False) # get stepsize/maximum value delta = cfg.a_stepsize xmax = delta * (2.**cfg.a_bitwidth - 1) if cfg.a_quantize is not None and 'pow2' in cfg.a_quantize: xmax = 2.**np.round(np.log2(xmax)) xmin = xmax / 2.**(2.**(cfg.a_bitwidth - 1) - 1) xmin = np.clip(xmin, cfg.a_xmin_min + 1e-5, cfg.a_xmin_max - 1e-5) xmax = np.clip(xmax, cfg.a_xmax_min + 1e-5, cfg.a_xmax_max - 1e-5) print(f'We use default delta ({delta, xmax}) for quantized nonlinearity.') if cfg.a_quantize == 'fp_relu': return F.fixed_point_quantize(x, sign=False, n=cfg.a_bitwidth, delta=cfg.a_stepsize) elif cfg.a_quantize == 'parametric_fp_b_xmax_relu': return PQ.parametric_fixed_point_quantize_b_xmax( x, sign=False, n_init=cfg.a_bitwidth, n_min=cfg.a_bitwidth_min, n_max=cfg.a_bitwidth_max, xmax_init=xmax, xmax_min=cfg.a_xmax_min, xmax_max=cfg.a_xmax_max, name='Aquant') elif cfg.a_quantize == 'parametric_fp_d_xmax_relu': return PQ.parametric_fixed_point_quantize_d_xmax( x, sign=False, d_init=delta, d_min=cfg.a_stepsize_min, d_max=cfg.a_stepsize_max, xmax_init=xmax, xmax_min=cfg.a_xmax_min, xmax_max=cfg.a_xmax_max, name='Aquant') elif cfg.a_quantize == 'parametric_fp_d_b_relu': return PQ.parametric_fixed_point_quantize_d_b(x, sign=False, n_init=cfg.a_bitwidth, n_min=cfg.a_bitwidth_min, n_max=cfg.a_bitwidth_max, d_init=delta, d_min=cfg.a_stepsize_min, d_max=cfg.a_stepsize_max, name='Aquant') elif cfg.a_quantize == 'pow2_relu': return F.pow2_quantize(x, sign=False, with_zero=True, n=cfg.a_bitwidth, m=np.round(np.log2(xmax))) elif cfg.a_quantize == 'parametric_pow2_b_xmax_relu': return PQ.parametric_pow2_quantize_b_xmax(x, sign=False, with_zero=True, n_init=cfg.a_bitwidth, n_min=cfg.a_bitwidth_min, n_max=cfg.a_bitwidth_max, xmax_init=xmax, xmax_min=cfg.a_xmax_min, xmax_max=cfg.a_xmax_max, name='Aquant') elif cfg.a_quantize == 'parametric_pow2_b_xmin_relu': return PQ.parametric_pow2_quantize_b_xmin(x, sign=False, with_zero=True, n_init=cfg.a_bitwidth, n_min=cfg.a_bitwidth_min, n_max=cfg.a_bitwidth_max, xmin_init=xmin, xmin_min=cfg.a_xmin_min, xmin_max=cfg.a_xmax_max, name='Aquant') elif cfg.a_quantize == 'parametric_pow2_xmin_xmax_relu': return PQ.parametric_pow2_quantize_xmin_xmax(x, sign=False, with_zero=True, xmin_init=xmin, xmin_min=cfg.a_xmin_min, xmin_max=cfg.a_xmax_max, xmax_init=xmax, xmax_min=cfg.a_xmax_min, xmax_max=cfg.a_xmax_max, name='Aquant') else: return F.relu(x, inplace=inplace)
def cifar10_fp_net_resnet23_prediction(image, maps=64, test=False): """ Construct Fixed-Point Net using resnet23. Fixed-Point Net quantizes weights and activations using FixedPointQuantize function. """ # Residual Unit def res_unit(x, scope_name, rng, dn=False, test=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> FixedPointQuantize -> Relu with nn.parameter_scope("conv1"): h = PF.fixed_point_quantized_convolution(x, C / 2, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.fixed_point_quantize(h) h = F.relu(h) # Conv -> BN -> FixedPointQuantize -> Relu with nn.parameter_scope("conv2"): h = PF.fixed_point_quantized_convolution(h, C / 2, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.fixed_point_quantize(h) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): h = PF.fixed_point_quantized_convolution(h, C, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> FixedPointQuantize -> Relu h = F.fixed_point_quantize(h) h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h ncls = 10 # Conv -> BN -> FixedPointQuantize with nn.parameter_scope("conv1"): # Preprocess image /= 255.0 if not test: image = F.image_augmentation(image, contrast=1.0, angle=0.25, flip_lr=True) image.need_grad = False h = PF.fixed_point_quantized_convolution(image, maps, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.fixed_point_quantize(h) h = res_unit(h, "conv2", False) # -> 32x32 h = res_unit(h, "conv3", True) # -> 16x16 h = res_unit(h, "conv4", False) # -> 16x16 h = res_unit(h, "conv5", True) # -> 8x8 h = res_unit(h, "conv6", False) # -> 8x8 h = res_unit(h, "conv7", True) # -> 4x4 h = res_unit(h, "conv8", False) # -> 4x4 h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1 pred = PF.fixed_point_quantized_affine(h, ncls) return pred