def __init__(self, num_features, eps=1e-5): super().__init__() self.num_features = num_features self.eps = eps self.weight = Parameter(np.ones(num_features, dtype=np.float32)) self.bias = Parameter(np.zeros(num_features, dtype=np.float32)) self.running_mean = Parameter( np.zeros((1, num_features, 1, 1), dtype=np.float32)) self.running_var = Parameter( np.ones((1, num_features, 1, 1), dtype=np.float32))
def test_elemwise_fuse_in_grad(trace_mode): w = Parameter(np.ones([4, 6]), dtype="float32") gm = GradManager().attach(w) opt = optim.SGD([w], lr=0.01, momentum=0.9, weight_decay=5e-4) # explicitly declare opt_level as 2 @trace(symbolic=trace_mode, opt_level=2) def f(): with gm: wm = F.sum(w**2, axis=1)**0.5 loss = wm.mean() gm.backward(loss) opt.step().clear_grad() return loss for i in range(3): y = f() y.numpy()
def run( N, IC, OC, IH, IW, KH, KW, PH, PW, SH, SW, has_bias=True, nonlinear_mode="identity", ): inp_v = np.random.normal(size=(N, IC, IH, IW)) w_v = np.random.normal(size=(OC, IC, KH, KW)) b_v = np.random.normal(size=(1, OC, 1, 1)) inp_scale = dtype.get_scale(inp_dtype) w_scale = dtype.get_scale(w_dtype) b_scale = dtype.get_scale(b_dtype) inpv = dtype.convert_to_qint8(inp_v * inp_scale, inp_dtype) wv = dtype.convert_to_qint8(w_v * w_scale, w_dtype) bv = dtype.convert_to_qint32(b_v * b_scale, b_dtype) inp_int8 = tensor(inpv, dtype=inp_dtype) w_int8 = Parameter(wv, dtype=w_dtype) b_int32 = Parameter(bv, dtype=b_dtype) inp_fp32 = inp_int8.astype("float32") w_fp32 = w_int8.astype("float32") b_fp32 = b_int32.astype("float32") def convert_to_nchw4(var): var = F.reshape(var, (var.shape[0], var.shape[1] // 4, 4, var.shape[2], var.shape[3])) var = F.transpose(var, (0, 1, 3, 4, 2)) return var def run_conv2d(inp, w, b): O = F.conv2d( inp, w, b if has_bias else None, stride=(SH, SW), padding=(PH, PW), ) if nonlinear_mode == "relu": return F.relu(O) else: return O def run_conv_bias(inp, w, b, format="NCHW"): b = b if has_bias else Parameter(np.zeros_like(b.numpy())) if format == "NCHW4": inp = convert_to_nchw4(inp) w = convert_to_nchw4(w) b = convert_to_nchw4(b) return F.quantized.conv_bias_activation( inp, w, b, stride=(SH, SW), padding=(PH, PW), dtype=out_dtype, nonlinear_mode=nonlinear_mode, ) format = "NCHW4" if is_cuda_available() else "NCHW" expected = run_conv2d(inp_fp32, w_fp32, b_fp32) expected = expected.astype(out_dtype).astype("float32") result = run_conv_bias(inp_int8, w_int8, b_int32, format=format).astype("float32") if format == "NCHW4": result = F.transpose(result, (0, 1, 4, 2, 3)) expected = F.flatten(expected) result = F.flatten(result) np.testing.assert_allclose(result.numpy(), expected.numpy(), atol=outp_scale)
def __init__(self, param_shape): super().__init__() self.params = [ Parameter(np.ones(param_shape), dtype=np.float32) for i in range(10) ]
def test_conv(module, padding_mode): normal_net = getattr(Float, module)(3, 3, 3, 1, 1, 1, bias=True, padding_mode=padding_mode) normal_net.eval() qat_net = getattr(QAT, module)(3, 3, 3, 1, 1, 1, bias=True, padding_mode=padding_mode) qat_net.eval() disable_observer(qat_net) propagate_qconfig(qat_net, min_max_fakequant_qconfig) init_qat_net(qat_net) x = mge.tensor(np.random.normal(size=(1, 3, 3, 3)).astype("float32")) inp_scale = gen_inp_scale() x = fake_quant_act(x, inp_scale) x.qparams.update(create_qparams(QuantMode.SYMMERTIC, "qint8", inp_scale)) x_int8 = quant(x, inp_scale) weight = np.random.normal(size=(3, 3, 3, 3)).astype("float32") bias = np.random.normal(size=(1, 3, 1, 1)).astype("float32") if module in ("ConvBn2d", "ConvBnRelu2d"): normal_net.conv.weight[...] = fake_quant_weight(weight, weight_scale) normal_net.conv.bias[...] = fake_quant_bias(bias, inp_scale * weight_scale) qat_net.conv.weight[...] = Parameter(weight) qat_net.conv.bias[...] = Parameter(bias) else: normal_net.weight[...] = fake_quant_weight(weight, weight_scale) normal_net.bias[...] = fake_quant_bias(bias, inp_scale * weight_scale) qat_net.weight[...] = Parameter(weight) qat_net.bias[...] = Parameter(bias) qat_from_float = getattr(QAT, module).from_float_module(normal_net) qat_from_float.eval() disable_observer(qat_from_float) disable_fake_quant(qat_from_float) q_net = getattr(Q, module).from_qat_module(qat_net) q_net.eval() normal = normal_net(x) qat_without_fakequant = qat_from_float(x) fake_quant_normal = fake_quant_act(normal_net(x), act_scale) qat = qat_net(x) q = q_net(x_int8).numpy() * act_scale np.testing.assert_allclose(qat_without_fakequant, normal, atol=1e-5) np.testing.assert_allclose(qat, fake_quant_normal, atol=act_scale) np.testing.assert_allclose(q, fake_quant_normal.numpy(), atol=act_scale)
def __init__(self): super().__init__() self.a = Parameter([1.0], dtype=np.float32)
def __init__(self): super().__init__() self.i = self.InnerModule() self.bn = BatchNorm2d(4) self.param = Parameter(np.ones(1, dtype=np.float32)) self.buff = Tensor(np.ones(1, dtype=np.float32))
def run( N, IC, OC, IH, IW, KH, KW, PH, PW, SH, SW, has_bias=True, nonlinear_mode="IDENTITY", ): inp_v = np.random.normal(size=(N, IC, IH, IW)) w_v = np.random.normal(size=(OC, IC, KW, KW)) b_v = np.random.normal(size=(1, OC, 1, 1)) inp_scale = mgb.dtype.get_scale(inp_dtype) w_scale = mgb.dtype.get_scale(w_dtype) b_scale = mgb.dtype.get_scale(b_dtype) inpv = mgb.dtype.convert_to_qint8(inp_v * inp_scale, inp_dtype) wv = mgb.dtype.convert_to_qint8(w_v * w_scale, w_dtype) bv = mgb.dtype.convert_to_qint32(b_v * b_scale, b_dtype) inp_int8 = tensor(inpv, dtype=inp_dtype) w_int8 = Parameter(wv, dtype=w_dtype) b_int32 = Parameter(bv, dtype=b_dtype) inp_fp32 = inp_int8.astype("float32") w_fp32 = w_int8.astype("float32") b_fp32 = b_int32.astype("float32") jit.trace.enabled = True b_symbolic = True def convert_to_nchw4(var): return var.reshape(var.shapeof(0), var.shapeof(1) // 4, 4, var.shapeof(2), var.shapeof(3)).dimshuffle(0, 1, 3, 4, 2) @jit.trace(symbolic=b_symbolic) def run_conv2d(inp, w, b): O = F.conv2d( inp, w, b if has_bias else None, stride=(SH, SW), padding=(PH, PW), ) if nonlinear_mode == "RELU": return F.relu(O) else: return O @jit.trace(symbolic=b_symbolic) def run_conv_bias(inp, w, b, format="NCHW"): b = b if has_bias else np.zeros_like(b) if format == "NCHW4": inp = convert_to_nchw4(inp) w = convert_to_nchw4(w) b = F.flatten(b) return F.conv_bias_activation( inp, w, b, stride=(SH, SW), padding=(PH, PW), dtype=out_dtype, nonlinear_mode=nonlinear_mode, ) format = "NCHW4" if is_cuda_available() else "NCHW" expected = run_conv2d(inp_fp32, w_fp32, b_fp32) expected = expected.astype(out_dtype).astype("float32") result = run_conv_bias(inp_int8, w_int8, b_int32, format=format).astype("float32") if format == "NCHW4": result = result.dimshuffle(0, 1, 4, 2, 3) expected = F.flatten(expected) result = F.flatten(result) assertTensorClose(result.numpy(), expected.numpy())
def __init__(self): super().__init__() self.a = Parameter([1.23], dtype="float32")
def __init__(self): super().__init__() self.params = [Parameter(1.0, dtype=np.float32) for i in range(10)]
def __init__(self, hidden_size, eps=1e-12): super(BertLayerNorm, self).__init__() self.weight = Parameter(np.ones(hidden_size).astype(np.float32)) self.bias = Parameter(np.zeros(hidden_size).astype(np.float32)) self.variance_epsilon = eps
def __init__(self, a, b): super().__init__() self.a = Parameter(a, dtype=np.float32) self.b = Parameter(b, dtype=np.float32) self.layer1 = MulFunc()
def __init__(self, a): super().__init__() self.a = Parameter(a, dtype=np.float32) self.layer = StopGradient()
def __init__(self, a): super().__init__() self.a = Parameter(a, dtype=np.float32) self.layer1 = STE()
def __init__(self, a, b): super().__init__() self.a = Parameter(a, dtype=np.float32) self.b = Parameter(b, dtype=np.float32) self.layer1 = EqWithFakeGrad()
def __init__(self, name): super().__init__() self.name = name self.k = Parameter(2.0, name="k")