def __init__(self, num_bits=8, quant_delay=0, symmetric=False, narrow_range=False): super(ReLU6Quant, self).__init__() self.fake_quant_act = nn.FakeQuantWithMinMax(min_init=0, max_init=6, num_bits=num_bits, quant_delay=quant_delay, ema=True, symmetric=symmetric, narrow_range=narrow_range) self.relu6 = P.ReLU6()
def __init__(self, num_bits=8, quant_delay=0, symmetric=False, narrow_range=False): super(HSigmoidQuant, self).__init__() self.fake_quant_act_before = nn.FakeQuantWithMinMax(min_init=0, max_init=6, num_bits=num_bits, quant_delay=quant_delay, ema=True, symmetric=symmetric, narrow_range=narrow_range) self.fake_quant_act_after = nn.FakeQuantWithMinMax(min_init=0, max_init=6, num_bits=num_bits, quant_delay=quant_delay, ema=True, symmetric=symmetric, narrow_range=narrow_range) self.act = P.HSigmoid()
def __init__(self, num_bits=8, quant_delay=0, symmetric=False, narrow_range=False): super(TensorAddQuant, self).__init__() self.fake_quant_act = nn.FakeQuantWithMinMax(min_init=-6, max_init=6, num_bits=num_bits, quant_delay=quant_delay, ema=True, symmetric=symmetric, narrow_range=narrow_range) self.add = P.TensorAdd()
def __init__(self, in_channels, out_channels, weight_init='normal', bias_init='zeros', has_bias=True, activation=None, num_bits=8, quant_delay=0, per_channel=False, symmetric=False, narrow_range=False): super(DenseQuant, self).__init__() self.in_channels = check_int_positive(in_channels) self.out_channels = check_int_positive(out_channels) self.has_bias = check_bool(has_bias) if isinstance(weight_init, Tensor): if weight_init.dim() != 2 or weight_init.shape()[0] != out_channels or \ weight_init.shape()[1] != in_channels: raise ValueError("weight_init shape error") self.weight = Parameter(initializer(weight_init, [out_channels, in_channels]), name="weight") if self.has_bias: if isinstance(bias_init, Tensor): if bias_init.dim() != 1 or bias_init.shape( )[0] != out_channels: raise ValueError("bias_init shape error") self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias") self.matmul = P.MatMul(transpose_b=True) self.bias_add = P.BiasAdd() self.activation = get_activation(activation) self.activation_flag = self.activation is not None self.fake_quant_weight = nn.FakeQuantWithMinMax( min_init=-6, max_init=6, ema=False, num_bits=num_bits, quant_delay=quant_delay, per_channel=per_channel, channel_size=out_channels, symmetric=symmetric, narrow_range=narrow_range)
def __init__(self, inp, oup, stride, expand_ratio): super(InvertedResidual, self).__init__() assert stride in [1, 2] hidden_dim = int(round(inp * expand_ratio)) self.use_res_connect = stride == 1 and inp == oup layers = [] if expand_ratio != 1: layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1)) layers.extend([ # dw ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim), # pw-linear nn.Conv2dBnFoldQuant(hidden_dim, oup, kernel_size=1, stride=1, pad_mode='pad', padding=0, group=1, per_channel=_per_channel, symmetric=_symmetric, quant_delay=_quant_delay), nn.FakeQuantWithMinMax(ema=True, ema_decay=_ema_decay, quant_delay=_quant_delay) ]) self.conv = nn.SequentialCell(layers) self.add = P.TensorAdd() self.add_fake = nn.FakeQuantWithMinMax(ema=True, ema_decay=_ema_decay, quant_delay=_quant_delay)
def __init__(self, block, layer_nums, in_channels, out_channels, strides, num_classes): super(ResNet, self).__init__() if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: raise ValueError( "the length of layer_num, in_channels, out_channels list must be 4!" ) self.conv1 = ConvBNReLU(3, 64, kernel_size=7, stride=2) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same") self.layer1 = self._make_layer(block, layer_nums[0], in_channel=in_channels[0], out_channel=out_channels[0], stride=strides[0]) self.layer2 = self._make_layer(block, layer_nums[1], in_channel=in_channels[1], out_channel=out_channels[1], stride=strides[1]) self.layer3 = self._make_layer(block, layer_nums[2], in_channel=in_channels[2], out_channel=out_channels[2], stride=strides[2]) self.layer4 = self._make_layer(block, layer_nums[3], in_channel=in_channels[3], out_channel=out_channels[3], stride=strides[3]) self.mean = P.ReduceMean(keep_dims=True) self.flatten = nn.Flatten() self.end_point = nn.DenseQuant(out_channels[3], num_classes, has_bias=True, per_channel=_per_channel, symmetric=_symmetric) self.output_fake = nn.FakeQuantWithMinMax(ema=True, ema_decay=_ema_decay) # init weights self._initialize_weights()
def __init__(self, in_channels, out_channels, kernel_size, stride=1, pad_mode='same', padding=0, dilation=1, group=1, has_bias=False, weight_init='normal', bias_init='zeros', quant_delay=0, num_bits=8, per_channel=False, symmetric=False, narrow_range=False): kernel_size = twice(kernel_size) super(Conv2dQuant, self).__init__(in_channels, out_channels, kernel_size, stride, pad_mode, padding, dilation, group, has_bias, weight_init, bias_init) self.conv2d = P.Conv2D(out_channel=self.out_channels, kernel_size=self.kernel_size, mode=1, pad_mode=self.pad_mode, pad=self.padding, stride=self.stride, dilation=self.dilation, group=self.group) self.bias_add = P.BiasAdd() if pad_mode not in ('valid', 'same', 'pad'): raise ValueError( 'Attr \'pad_mode\' of \'Conv2d\' Op passed ' + str(pad_mode) + ', should be one of values in \'valid\', \'same\', \'pad\'.') self.fake_quant_weight = nn.FakeQuantWithMinMax( min_init=-6, max_init=6, ema=False, num_bits=num_bits, quant_delay=quant_delay, per_channel=per_channel, channel_size=out_channels, symmetric=symmetric, narrow_range=narrow_range)
def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1): super(ConvBNReLU, self).__init__() padding = (kernel_size - 1) // 2 conv = nn.Conv2dBnFoldQuant(in_planes, out_planes, kernel_size, stride, pad_mode='pad', padding=padding, quant_delay=_quant_delay, group=groups, per_channel=_per_channel, symmetric=_symmetric) layers = [conv, nn.ReLU()] self.features = nn.SequentialCell(layers) self.fake = nn.FakeQuantWithMinMax(ema=True, ema_decay=_ema_decay, min_init=0, quant_delay=_quant_delay)
def __init__(self, in_channels, out_channels, kernel_size, stride, pad_mode, padding=0, dilation=1, group=1, eps=1e-5, momentum=0.9, weight_init=None, beta_init=None, gamma_init=None, mean_init=None, var_init=None, quant_delay=0, freeze_bn=100000, fake=True, num_bits=8, per_channel=False, symmetric=False, narrow_range=False): super(Conv2dBatchNormQuant, self).__init__() _ = dilation self.stride = stride self.conv = P.Conv2D(out_channel=out_channels, kernel_size=kernel_size, mode=1, pad_mode=pad_mode, pad=padding, stride=stride, dilation=1, group=group) self.fake = fake self.freeze_bn = freeze_bn if isinstance(kernel_size, int): kernel_size = (kernel_size, kernel_size) if weight_init is None: weight_init = initializer( 'normal', [out_channels, in_channels // group, *kernel_size]) self.weight = Parameter(weight_init, name='weight') if gamma_init is None: gamma_init = initializer('ones', [out_channels]) self.gamma = Parameter(gamma_init, name='gamma') if beta_init is None: beta_init = initializer('zeros', [out_channels]) self.beta = Parameter(beta_init, name='beta') if mean_init is None: mean_init = initializer('zeros', [out_channels]) self.moving_mean = Parameter( mean_init, name='moving_mean', requires_grad=False) if var_init is None: var_init = initializer('ones', [out_channels]) self.moving_variance = Parameter( var_init, name='moving_variance', requires_grad=False) self.step = Parameter(initializer( 'normal', [1], dtype=mstype.int32), name='step', requires_grad=False) self.fake_quant_weight = nn.FakeQuantWithMinMax(min_init=-6, max_init=6, ema=False, num_bits=num_bits, quant_delay=quant_delay, per_channel=per_channel, channel_size=out_channels, symmetric=symmetric, narrow_range=narrow_range) self.batchnorm_fold_train = P.BatchNormFold(epsilon=eps, momentum=momentum, is_training=True, freeze_bn=freeze_bn) self.batchnorm_fold_infer = P.BatchNormFold(epsilon=eps, momentum=momentum, is_training=False, freeze_bn=freeze_bn) self.correct_mul = P.CorrectionMul() self.relu = P.ReLU() self.batchnorm_fold2 = P.BatchNormFold2(freeze_bn=freeze_bn) self.batchnorm_fold2_infer = P.BatchNormFold2(freeze_bn=0) self.one = Tensor(1, mstype.int32) self.assignadd = P.AssignAdd()
def __init__(self, num_classes=1000, width_mult=1., has_dropout=False, inverted_residual_setting=None, round_nearest=8): super(MobileNetV2Quant, self).__init__() block = InvertedResidual input_channel = 32 last_channel = 1280 # setting of inverted residual blocks self.cfgs = inverted_residual_setting if inverted_residual_setting is None: self.cfgs = [ # t, c, n, s [1, 16, 1, 1], [6, 24, 2, 2], [6, 32, 3, 2], [6, 64, 4, 2], [6, 96, 3, 1], [6, 160, 3, 2], [6, 320, 1, 1], ] # building first layer input_channel = _make_divisible(input_channel * width_mult, round_nearest) self.out_channels = _make_divisible( last_channel * max(1.0, width_mult), round_nearest) self.input_fake = nn.FakeQuantWithMinMax(ema=True, ema_decay=_ema_decay, quant_delay=_quant_delay) features = [ConvBNReLU(3, input_channel, stride=2)] # building inverted residual blocks for t, c, n, s in self.cfgs: output_channel = _make_divisible(c * width_mult, round_nearest) for i in range(n): stride = s if i == 0 else 1 features.append( block(input_channel, output_channel, stride, expand_ratio=t)) input_channel = output_channel # building last several layers features.append( ConvBNReLU(input_channel, self.out_channels, kernel_size=1)) # make it nn.CellList self.features = nn.SequentialCell(features) # mobilenet head head = ([ GlobalAvgPooling(), nn.DenseQuant(self.out_channels, num_classes, has_bias=True, per_channel=_per_channel, symmetric=_symmetric, quant_delay=_quant_delay), nn.FakeQuantWithMinMax(ema=True, ema_decay=_ema_decay) ] if not has_dropout else [ GlobalAvgPooling(), nn.Dropout(0.2), nn.DenseQuant(self.out_channels, num_classes, has_bias=True, per_channel=_per_channel, symmetric=_symmetric, quant_delay=_quant_delay), nn.FakeQuantWithMinMax( ema=True, ema_decay=_ema_decay, quant_delay=_quant_delay) ]) self.head = nn.SequentialCell(head)