def generate(self, num_samples):
"""Grabs input/label pairs from MNIST"""
ctx = mxnet.cpu()
# Load a random image from the test dataset
sample_data = mxnet.gluon.data.DataLoader(
mxnet.gluon.data.vision.CIFAR10(train=False),
1,
shuffle=self.config.get('shuffle', True))
samples = []
for i, (data, label) in zip(range(num_samples), sample_data):
if i == num_samples:
break
data_np = numpy.copy(data.as_in_context(ctx).asnumpy())
# gluon data is in NHWC format
shape = util.LabelledShape(dim_iter=zip('NHWC', data_np.shape),
dtype='uint8')
data_nt = util.LabelledTensor(data_np,
shape).with_layout(self.DATA_LAYOUT)
label = int(label.asnumpy()[0])
samples.append(
DatasetSample(inputs={'data': data_nt},
outputs={'label': label}))
return samples
def _load_cmsis_params(mod, param_file, param_shapes):
with open(param_file) as f:
cmsis_params = json.load(f)
params = {}
for formal_param in mod['main'].params[1:]: # exclude data
name = formal_param.name_hint
_LOG.debug('name %r %r %r', name, _CMSIS_PARAM_SHAPES[name].dtype,
len(cmsis_params[name]))
cmsis_tensor = util.LabelledTensor(data=np.array(
cmsis_params[name],
dtype=_CMSIS_PARAM_SHAPES[name].dtype,
copy=True).reshape(_CMSIS_PARAM_SHAPES[name].shape),
shape=_CMSIS_PARAM_SHAPES[name])
if name == 'conv0_bias':
cmsis_tensor = _CMSIS_PARAM_SHAPES[name].gen_zero_tensor()
param_shape = param_shapes[name]
relay_shape = util.LabelledShape(
zip(param_shape.layout,
[x.value for x in formal_param.checked_type.shape]),
dtype=param_shape.dtype)
assert param_shape.dims == relay_shape.dims
param = cmsis_tensor.resize(param_shape)
data = param.data
if name == 'mean_data':
data = relay_shape.gen_zero_tensor().data
params[name] = tvm.nd.array(data, tvm.cpu(0))
return params
def get_sample_points(n, data_layout='NHWC'):
"""Grabs a single input/label pair from MNIST"""
ctx = mxnet.cpu()
# Load a random image from the test dataset
sample_data = mxnet.gluon.data.DataLoader(
mxnet.gluon.data.vision.CIFAR10(train=False),
1)#,
# shuffle=True)
samples = []
for i, (data, label) in zip(range(n), sample_data):
if i == n:
break
data_np = numpy.copy(data.as_in_context(ctx).asnumpy())
# gluon data is in NHWC format
shape = util.LabelledShape(dim_iter=zip('NHWC', data_np.shape), dtype='uint8')
data_nt = util.LabelledTensor(data_np, shape).with_layout(data_layout)
label = int(label.asnumpy()[0])
samples.append({'data': data_nt, 'label': label})
return samples
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--onnx-model',
default=f'{util.get_repo_root()}/data/cifar10.onnx',
help='path to unquantized cifar10 model')
parser.add_argument(
'--num-samples',
default=1024,
type=int,
help='number of samples to use for data-aware quantization')
parser.add_argument(
'--quantized-tvm-model',
default=f'{util.get_repo_root()}/data/quantized-cifar10.onnx',
help='path to write quantized cifar10 model')
args = parser.parse_args()
onnx_model = onnx.load(args.onnx_model)
data_shape = util.LabelledShape(N=1, C=3, H=32, W=32, dtype='uint8')
mod, params = relay.frontend.from_onnx(onnx_model,
{"data": data_shape.shape})
samples = model_util.get_sample_points(args.num_samples, data_shape.layout)
numpy_samples = []
for data in samples:
numpy_samples.append({'data': data['data'].data})
with relay.quantize.qconfig(calibrate_mode='global_scale',
global_scale=8.0,
nbit_activation=8,
dtype_activation="int8",
skip_conv_layers=[0],
dtype_input="int8",
dtype_weight="int8"):
quantized = relay.quantize.quantize(mod,
params) #, dataset=numpy_samples)
print('output:', quantized)
def build_model(self):
kernel_layouts = self._kernel_layouts
# TODO change relay/op/tensor/unary.cc _make.clip to accept exprs instead of doubles
# TODO discrepancies between outputs might be a result of the bias_add op
# not matching the semantics of the CMSIS bias add.
data_shape = util.LabelledShape.from_dims_and_layout(dict(N=1,
C=3,
H=32,
W=32),
self.DATA_LAYOUT,
dtype='uint8')
conv0_weight_shape = util.LabelledShape.from_dims_and_layout(
dict(H=5, W=5, I=3, O=32), kernel_layouts[0], dtype='int8')
if self._is_simd:
# to fit our SIMD intrinsic, we make the 'C' dimension a multiple of 4
data_shape = util.LabelledShape.from_dims_and_layout(
dict(N=1, C=4, H=32, W=32), self.DATA_LAYOUT, dtype='uint8')
conv0_weight_shape = util.LabelledShape.from_dims_and_layout(
dict(H=5, W=5, O=32, I=4), kernel_layouts[0], dtype='int8')
_LOG.debug('data_shape %r', data_shape)
_LOG.debug('conv0_weight_shape %r', conv0_weight_shape)
param_shapes = collections.OrderedDict([
('data', data_shape),
('mean_data', data_shape),
('conv0_weight', conv0_weight_shape),
('conv0_bias', util.LabelledShape(B=32, dtype='int8')),
('conv1_weight',
util.LabelledShape.from_dims_and_layout(dict(O=32, I=32, H=5,
W=5),
kernel_layouts[1],
dtype='int8')),
('conv1_bias', util.LabelledShape(B=32, dtype='int8')),
('conv2_weight',
util.LabelledShape.from_dims_and_layout(dict(O=64, I=32, H=5,
W=5),
kernel_layouts[2],
dtype='int8')),
('conv2_bias', util.LabelledShape(B=64, dtype='int8')),
('dense0_weight', util.LabelledShape(O=10, I=1024, dtype='int8')),
('dense0_bias', util.LabelledShape(B=10, dtype='int8')),
])
bias_add_axis = self.DATA_LAYOUT.index('C')
params = []
for p, s in param_shapes.items():
joined_shape = ', '.join(str(x) for x in s.shape)
params.append(f' %{p}: Tensor[({joined_shape}), {s.dtype}]')
param_args = ',\n'.join(params)
_LOG.debug('params %s', param_args)
mod = tvm.relay.fromtext(
CIFAR10_RELAY_MODEL.format(bias_add_axis=bias_add_axis,
data_layout=self.DATA_LAYOUT,
kernel_layouts=kernel_layouts,
param_args=param_args))
if self.config.get('use_random_params', True):
params = _gen_random_params(mod, param_shapes)
else:
params = _load_cmsis_params(mod,
self.config.relpath('parameter_file'),
param_shapes)
return CompiledModel(self.target, mod, params, 'main', {
'data_layout': self.DATA_LAYOUT,
'kernel_layouts': kernel_layouts
})
if _cache_key not in GENERATED_RANDOM_PARAMS:
# generate random params
params = {}
for param in mod['main'].params[1:]:
name = param.name_hint
low, high = _RANDOM_BOUNDS[name]
rand_tensor = param_shapes[name].gen_rand_tensor(low, high)
params[param.name_hint] = rand_tensor.data
GENERATED_RANDOM_PARAMS[_cache_key] = params
return GENERATED_RANDOM_PARAMS[_cache_key]
_CMSIS_PARAM_SHAPES = {
'mean_data': util.LabelledShape(N=1, H=32, W=32, C=3, dtype='uint8'),
'conv0_weight': util.LabelledShape(O=32, H=5, W=5, I=3, dtype='int8'),
'conv0_bias': util.LabelledShape(B=32, dtype='int8'),
'conv1_weight': util.LabelledShape(O=32, H=5, W=5, I=32, dtype='int8'),
'conv1_bias': util.LabelledShape(B=32, dtype='int8'),
'conv2_weight': util.LabelledShape(O=64, H=5, W=5, I=32, dtype='int8'),
'conv2_bias': util.LabelledShape(B=64, dtype='int8'),
'dense0_weight': util.LabelledShape(O=10, I=1024, dtype='int8'),
'dense0_bias': util.LabelledShape(B=10, dtype='int8'),
}
def _load_cmsis_params(mod, param_file, param_shapes):
with open(param_file) as f:
cmsis_params = json.load(f)
def build_model(self):
mod = mock_c_mod.build([CIFAR10_IMPLS[self.impl]['src']], 'cifar10', [
('data', util.LabelledShape(N=1, H=32, W=32, C=3, dtype='uint8'))
], util.LabelledShape(N=1, X=10, dtype='int8'))
return CompiledModel(self.target, mod, [], 'cifar10', {})
