def res_unit(x, scope_name, rng, dn=False, test=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> Pow2Quantize -> Relu with nn.parameter_scope("conv1"): h = PF.pow2_quantized_convolution(x, C / 2, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.pow2_quantize(h) h = F.relu(h) # Conv -> BN -> Pow2Quantize -> Relu with nn.parameter_scope("conv2"): h = PF.pow2_quantized_convolution(h, C / 2, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.pow2_quantize(h) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): h = PF.pow2_quantized_convolution(h, C, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> Pow2Quantize -> Relu h = F.pow2_quantize(h) h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h
def test_pow2_quantize_forward_backward(seed, sign, with_zero, n, m, quantize, ste_fine_grained, ctx, func_name): from nbla_test_utils import cap_ignore_region, \ function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3)) ] if quantize: func_args = [sign, with_zero, n, m, quantize, ste_fine_grained] function_tester(rng, F.pow2_quantize, ref_pow2_quantize, inputs, func_args=func_args, atol_b=1e-3, backward=[True], ctx=ctx, func_name=func_name, ref_grad=ref_grad_pow2_quantize) else: # No quantize for i in inputs: v = nn.Variable(i.shape) v.d = i v.g = np.random.randn(*i.shape) o = F.pow2_quantize(v) o.forward() o.backward() np.allclose(v.d, o.d) np.allclose(v.g, o.g)
def nonl(x, cfg, inplace=False): # for convenience, store size of x (this allows us to compute the number of activations) _s = get_parameter_or_create('Asize', (), ConstantInitializer(np.prod(x.shape[1:])), need_grad=False) # get stepsize/maximum value delta = cfg.a_stepsize xmax = delta * (2.**cfg.a_bitwidth - 1) if cfg.a_quantize is not None and 'pow2' in cfg.a_quantize: xmax = 2.**np.round(np.log2(xmax)) xmin = xmax / 2.**(2.**(cfg.a_bitwidth - 1) - 1) xmin = np.clip(xmin, cfg.a_xmin_min + 1e-5, cfg.a_xmin_max - 1e-5) xmax = np.clip(xmax, cfg.a_xmax_min + 1e-5, cfg.a_xmax_max - 1e-5) print(f'We use default delta ({delta, xmax}) for quantized nonlinearity.') if cfg.a_quantize == 'fp_relu': return F.fixed_point_quantize(x, sign=False, n=cfg.a_bitwidth, delta=cfg.a_stepsize) elif cfg.a_quantize == 'parametric_fp_b_xmax_relu': return PQ.parametric_fixed_point_quantize_b_xmax( x, sign=False, n_init=cfg.a_bitwidth, n_min=cfg.a_bitwidth_min, n_max=cfg.a_bitwidth_max, xmax_init=xmax, xmax_min=cfg.a_xmax_min, xmax_max=cfg.a_xmax_max, name='Aquant') elif cfg.a_quantize == 'parametric_fp_d_xmax_relu': return PQ.parametric_fixed_point_quantize_d_xmax( x, sign=False, d_init=delta, d_min=cfg.a_stepsize_min, d_max=cfg.a_stepsize_max, xmax_init=xmax, xmax_min=cfg.a_xmax_min, xmax_max=cfg.a_xmax_max, name='Aquant') elif cfg.a_quantize == 'parametric_fp_d_b_relu': return PQ.parametric_fixed_point_quantize_d_b(x, sign=False, n_init=cfg.a_bitwidth, n_min=cfg.a_bitwidth_min, n_max=cfg.a_bitwidth_max, d_init=delta, d_min=cfg.a_stepsize_min, d_max=cfg.a_stepsize_max, name='Aquant') elif cfg.a_quantize == 'pow2_relu': return F.pow2_quantize(x, sign=False, with_zero=True, n=cfg.a_bitwidth, m=np.round(np.log2(xmax))) elif cfg.a_quantize == 'parametric_pow2_b_xmax_relu': return PQ.parametric_pow2_quantize_b_xmax(x, sign=False, with_zero=True, n_init=cfg.a_bitwidth, n_min=cfg.a_bitwidth_min, n_max=cfg.a_bitwidth_max, xmax_init=xmax, xmax_min=cfg.a_xmax_min, xmax_max=cfg.a_xmax_max, name='Aquant') elif cfg.a_quantize == 'parametric_pow2_b_xmin_relu': return PQ.parametric_pow2_quantize_b_xmin(x, sign=False, with_zero=True, n_init=cfg.a_bitwidth, n_min=cfg.a_bitwidth_min, n_max=cfg.a_bitwidth_max, xmin_init=xmin, xmin_min=cfg.a_xmin_min, xmin_max=cfg.a_xmax_max, name='Aquant') elif cfg.a_quantize == 'parametric_pow2_xmin_xmax_relu': return PQ.parametric_pow2_quantize_xmin_xmax(x, sign=False, with_zero=True, xmin_init=xmin, xmin_min=cfg.a_xmin_min, xmin_max=cfg.a_xmax_max, xmax_init=xmax, xmax_min=cfg.a_xmax_min, xmax_max=cfg.a_xmax_max, name='Aquant') else: return F.relu(x, inplace=inplace)
def cifar10_pow2_net_resnet23_prediction(image, maps=64, test=False): """ Construct Pow2 Net using resnet23. Pow2 Net quantizes weights and activations using Pow2Quantize function. """ # Residual Unit def res_unit(x, scope_name, rng, dn=False, test=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> Pow2Quantize -> Relu with nn.parameter_scope("conv1"): h = PF.pow2_quantized_convolution(x, C / 2, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.pow2_quantize(h) h = F.relu(h) # Conv -> BN -> Pow2Quantize -> Relu with nn.parameter_scope("conv2"): h = PF.pow2_quantized_convolution(h, C / 2, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.pow2_quantize(h) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): h = PF.pow2_quantized_convolution(h, C, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> Pow2Quantize -> Relu h = F.pow2_quantize(h) h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h ncls = 10 # Conv -> BN -> Pow2Quantize with nn.parameter_scope("conv1"): # Preprocess image /= 255.0 if not test: image = F.image_augmentation(image, contrast=1.0, angle=0.25, flip_lr=True) image.need_grad = False h = PF.pow2_quantized_convolution(image, maps, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.pow2_quantize(h) h = res_unit(h, "conv2", False) # -> 32x32 h = res_unit(h, "conv3", True) # -> 16x16 h = res_unit(h, "conv4", False) # -> 16x16 h = res_unit(h, "conv5", True) # -> 8x8 h = res_unit(h, "conv6", False) # -> 8x8 h = res_unit(h, "conv7", True) # -> 4x4 h = res_unit(h, "conv8", False) # -> 4x4 h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1 pred = PF.pow2_quantized_affine(h, ncls) return pred