def test_invalid_layout(op):
"""Checks CMSIS-NN partitioning when layout is not NHWC"""
model = make_model(pool_op=op, layout="NCHW")
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
assert_no_external_function(cmsisnn_mod)
def test_invalid_parameters(
op,
input_dtype,
):
input_scale = 0.256
input_zero_point = 33
model = make_model(
op,
[1, 16, 16, 3],
input_dtype,
input_dtype,
input_scale,
input_zero_point,
input_scale,
input_zero_point,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
attrs = [
cmsisnn_mod[var.name_hint].attrs
for var in cmsisnn_mod.get_global_vars()
if cmsisnn_mod[var.name_hint].attrs
]
assert not any(attrs), "No function should have an external attribute."
def test_op_int8(zero_point, scale, compiler_cpu, cpu_flags):
"""Tests int8 QNN Softmax for CMSIS-NN"""
interface_api = "c"
use_unpacked_api = True
dtype = "int8"
shape = [1, 16, 16, 3]
model = make_model(shape, dtype, dtype, zero_point, scale)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
# validate pattern matching
assert_partitioned_function(orig_mod, cmsisnn_mod)
# validate the output
in_min, in_max = get_range_for_dtype_str(dtype)
np.random.seed(0)
input_data = np.random.randint(in_min, high=in_max, size=shape, dtype=dtype)
inputs = {"in0": input_data}
params = {}
output_list = generate_ref_data(orig_mod["main"], inputs, params)
compile_and_run(
AOTTestModel(module=cmsisnn_mod, inputs=inputs, outputs=output_list, params=params),
create_test_runner(compiler_cpu, cpu_flags),
interface_api,
use_unpacked_api,
)
def test_op_int8(zero_point, scale):
interface_api = "c"
use_unpacked_api = True
test_runner = AOT_USMP_CORSTONE300_RUNNER
dtype = "int8"
shape = [1, 16, 16, 3]
model = make_model(shape, dtype, dtype, zero_point, scale)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
# validate pattern matching
assert_partitioned_function(orig_mod, cmsisnn_mod)
# validate the output
in_min, in_max = get_range_for_dtype_str(dtype)
np.random.seed(0)
input_data = np.random.randint(in_min,
high=in_max,
size=shape,
dtype=dtype)
inputs = {"in0": input_data}
params = {}
output_list = generate_ref_data(orig_mod["main"], inputs, params)
compile_and_run(
AOTTestModel(module=cmsisnn_mod,
inputs=inputs,
outputs=output_list,
params=params),
test_runner,
interface_api,
use_unpacked_api,
)
def test_conv2d_int8_tflite(ifm_shape, kernel_shape, strides, dilation,
padding, activation):
interface_api = "c"
use_unpacked_api = True
test_runner = AOT_USMP_CORSTONE300_RUNNER
dtype = "int8"
tflite_model, relay_mod, params = create_conv2d_tflite_relay_models(
ifm_shape, kernel_shape, strides, dilation, padding, activation, dtype)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(relay_mod, params)
# validate pattern matching
assert_partitioned_function(relay_mod, cmsisnn_mod)
# validate CMSIS-NN output against TFLite output
input_map, output_map, output_tolerance = generate_ref_data_tflite(
tflite_model)
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=input_map,
outputs=output_map,
params=params,
output_tolerance=output_tolerance,
),
test_runner,
interface_api,
use_unpacked_api,
)
def test_cnn_small():
# download the model
base_url = "https://github.com/ARM-software/ML-zoo/raw/master/models/keyword_spotting/cnn_small/tflite_int8"
file_to_download = "cnn_s_quantized.tflite"
model_file = download_testdata("{}/{}".format(base_url, file_to_download), file_to_download)
with open(model_file, "rb") as f:
tflite_model_buf = f.read()
input_shape = (1, 490)
in_min, in_max = get_range_for_dtype_str("int8")
input_data = np.random.randint(in_min, high=in_max, size=input_shape).astype(np.float32)
orig_mod, params = convert_to_relay(tflite_model_buf, input_data, "input")
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod, params)
# validate CMSIS-NN output against CPU output
interface_api = "c"
use_unpacked_api = True
test_runner = AOT_CORSTONE300_RUNNER
inputs = {"input": input_data}
params = {}
output_list = generate_ref_data(orig_mod["main"], inputs, params)
compile_and_run(
AOTTestModel(module=cmsisnn_mod, inputs=inputs, outputs=output_list, params=params),
test_runner,
interface_api,
use_unpacked_api,
)
def test_conv2d_int8_tflite(ifm_shape, kernel_shape, strides, dilation, padding, activation):
"""Compares TVM output against TFLite output"""
interface_api = "c"
use_unpacked_api = True
test_runner = AOT_USMP_CORSTONE300_RUNNER
dtype = "int8"
from tvm.relay.testing.tflite import TFLiteModel
tfl_model = TFLiteModel(dtype)
conv2d_function = tfl_model.create_conv2d_single(
kernel_shape, strides, padding, dilation, activation
)
tfl_model.create_tflite_model(conv2d_function, [ifm_shape])
relay_mod, relay_params = tfl_model.convert_to_relay()
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(relay_mod, relay_params)
# validate pattern matching
assert_partitioned_function(relay_mod, cmsisnn_mod)
# validate CMSIS-NN output against TFLite output
input_map, output_map, output_tolerance = tfl_model.generate_reference_data()
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=input_map,
outputs=output_map,
params=relay_params,
output_tolerance=output_tolerance,
),
test_runner,
interface_api,
use_unpacked_api,
)
def test_invalid_datatype(op):
"""Checks CMSIS-NN partitioning for non int8 dtype"""
model = make_model(pool_op=op, dtype="int64")
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
assert_no_external_function(cmsisnn_mod)
def test_invalid_parameters(in_dtype, out_dtype, zero_point, scale,
out_zero_point, out_scale):
model = make_model([1, 16, 16, 3], in_dtype, out_dtype, zero_point, scale,
out_zero_point, out_scale)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
assert_no_external_function(cmsisnn_mod)
def test_invalid_batch_size(op):
model = make_model(
pool_op=op,
shape=(2, 28, 28, 12),
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
assert_no_external_function(cmsisnn_mod)
def test_op_int8(
op,
relu_type,
input_0_scale,
input_0_zero_point,
input_1_scale,
input_1_zero_point,
compiler_cpu,
cpu_flags,
):
"""Tests QNN binary operator for CMSIS-NN"""
interface_api = "c"
use_unpacked_api = True
dtype = "int8"
shape = [1, 16, 16, 3]
model = make_model(
op,
generate_variable("input_0"),
generate_variable("input_1"),
input_0_scale,
input_0_zero_point,
input_1_scale,
input_1_zero_point,
relu_type,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
# validate pattern matching
assert_partitioned_function(orig_mod, cmsisnn_mod)
# validate the output
in_min, in_max = get_range_for_dtype_str(dtype)
inputs = {
"input_0": np.random.randint(in_min,
high=in_max,
size=shape,
dtype=dtype),
"input_1": np.random.randint(in_min,
high=in_max,
size=shape,
dtype=dtype),
}
output_list = generate_ref_data(orig_mod["main"], inputs)
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=inputs,
outputs=output_list,
output_tolerance=1,
),
create_test_runner(compiler_cpu, cpu_flags),
interface_api,
use_unpacked_api,
)
def test_invalid_parameters(in_dtype, out_dtype, zero_point, scale,
out_zero_point, out_scale):
"""Tests for non int8 input and output of Softmax"""
model = make_model([1, 16, 16, 3], in_dtype, out_dtype, zero_point, scale,
out_zero_point, out_scale)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
assert_no_external_function(cmsisnn_mod)
def test_op_int8(op, input_0_scale, input_0_zero_point, input_1_scale, input_1_zero_point):
interface_api = "c"
use_unpacked_api = True
test_runner = AOT_CORSTONE300_RUNNER
dtype = "int8"
shape = [1, 16, 16, 3]
model = make_model(
op,
shape,
dtype,
dtype,
input_0_scale,
input_0_zero_point,
input_1_scale,
input_1_zero_point,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
# validate pattern matching
attrs = [
cmsisnn_mod[var.name_hint].attrs
for var in cmsisnn_mod.get_global_vars()
if cmsisnn_mod[var.name_hint].attrs
]
assert any(attrs), "At least one function with external attributes was expected."
compilers = [
key == "Compiler" and value == "cmsis-nn" for attr in attrs for key, value in attr.items()
]
assert any(compilers), "Module does not contain function for cmsisnn target."
assert count_num_calls(orig_mod) == count_num_calls(
cmsisnn_mod
), "Number of calls changed during partitioning"
# validate the output
in_min, in_max = get_range_for_dtype_str(dtype)
inputs = {
"input_0": np.random.randint(in_min, high=in_max, size=shape, dtype=dtype),
"input_1": np.random.randint(in_min, high=in_max, size=shape, dtype=dtype),
}
output_list = generate_ref_data(orig_mod["main"], inputs)
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=inputs,
outputs=output_list,
output_tolerance=1,
),
test_runner,
interface_api,
use_unpacked_api,
)
def test_invalid_parameters(
in_dtype,
kernel_dtype,
kernel_zero_point,
padding,
):
ifm_shape = (1, 28, 28, 12)
out_channels = 2
input_scale = 1
input_zero_point = 24
kernel_scale = [0.11, 0.0237]
in_min, in_max = get_range_for_dtype_str(in_dtype)
kernel_layout = "HWIO"
kernel_shape = [3, 3, ifm_shape[3], out_channels]
output_scale, output_zero_point = get_conv2d_qnn_params(
kernel_shape,
input_scale,
input_zero_point,
kernel_scale,
kernel_zero_point,
in_dtype,
kernel_dtype,
in_dtype,
False,
)
model, params = make_model(
shape=ifm_shape,
kernel_shape=kernel_shape,
input_zero_point=input_zero_point,
input_scale=input_scale,
kernel_zero_point=kernel_zero_point,
kernel_scale=kernel_scale,
output_zero_point=output_zero_point,
output_scale=output_scale,
padding=padding,
strides=(1, 1),
dilation=(1, 1),
groups=1,
dtype=in_dtype,
kernel_dtype=kernel_dtype,
out_channels=out_channels,
weight_format=kernel_layout,
enable_bias=True,
relu_type="NONE",
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod, params)
# validate pattern matching
attrs = [
cmsisnn_mod[var.name_hint].attrs
for var in cmsisnn_mod.get_global_vars()
if cmsisnn_mod[var.name_hint].attrs
]
assert not any(attrs), "No function should have an external attribute."
def test_same_input_to_binary_op(op, relu_type):
"""Tests QNN binary operator for CMSIS-NN where both inputs are the same"""
interface_api = "c"
use_unpacked_api = True
test_runner = AOT_USMP_CORSTONE300_RUNNER
dtype = "int8"
shape = [1, 16, 16, 3]
input_ = generate_variable("input")
input_scale = 0.256
input_zero_point = 33
model = make_model(
op,
input_,
input_,
input_scale,
input_zero_point,
input_scale,
input_zero_point,
relu_type,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
# validate pattern matching
assert_partitioned_function(orig_mod, cmsisnn_mod)
# Check if the number of internal function parameter is 1
cmsisnn_global_func = cmsisnn_mod["tvmgen_default_cmsis_nn_main_0"]
assert (
isinstance(cmsisnn_global_func.body, tvm.relay.expr.Call)
and len(cmsisnn_global_func.body.args) == 1
), "Composite function for the binary op should have only 1 parameter."
# validate the output
in_min, in_max = get_range_for_dtype_str(dtype)
inputs = {
"input": np.random.randint(in_min,
high=in_max,
size=shape,
dtype=dtype),
}
output_list = generate_ref_data(orig_mod["main"], inputs)
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=inputs,
outputs=output_list,
output_tolerance=1,
),
test_runner,
interface_api,
use_unpacked_api,
)
def test_invalid_batch_size(op):
"""Checks CMSIS-NN partitioning when batch size is not 1"""
model = make_model(
pool_op=op,
shape=(2, 28, 28, 12),
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
assert_no_external_function(cmsisnn_mod)
def test_op_int8(
in_shape,
pool_size,
strides,
padding,
relu_type,
pool_type,
zero_point,
scale,
compiler_cpu,
cpu_flags,
):
"""Tests QNN pooling op for int8 inputs"""
interface_api = "c"
use_unpacked_api = True
dtype = "int8"
model = make_model(
pool_op=pool_type,
shape=in_shape,
pool_size=pool_size,
strides=strides,
padding=padding,
scale=scale,
zero_point=zero_point,
relu_type=relu_type,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
# validate pattern matching
assert_partitioned_function(orig_mod, cmsisnn_mod)
# validate the output
in_min, in_max = get_range_for_dtype_str(dtype)
np.random.seed(0)
inputs = {
"input":
np.random.randint(in_min, high=in_max, size=in_shape, dtype="int8"),
}
output_list = generate_ref_data(orig_mod["main"], inputs)
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=inputs,
outputs=output_list,
params=None,
output_tolerance=1,
),
create_test_runner(compiler_cpu, cpu_flags),
interface_api,
use_unpacked_api,
)
def test_op_int8(
in_shape,
pool_size,
strides,
padding,
relu_type,
pool_type,
zero_point,
scale,
):
interface_api = "c"
use_unpacked_api = True
test_runner = AOT_USMP_CORSTONE300_RUNNER
dtype = "int8"
model = make_model(
pool_type,
in_shape,
pool_size,
strides,
padding,
dtype,
scale,
zero_point,
relu_type,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
# validate pattern matching
assert_partitioned_function(orig_mod, cmsisnn_mod)
# validate the output
in_min, in_max = get_range_for_dtype_str(dtype)
np.random.seed(0)
inputs = {
"input":
np.random.randint(in_min, high=in_max, size=in_shape, dtype="int8"),
}
output_list = generate_ref_data(orig_mod["main"], inputs)
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=inputs,
outputs=output_list,
params=None,
output_tolerance=1,
),
test_runner,
interface_api,
use_unpacked_api,
)
def test_constant_input_int8(op, input_0, input_1):
"""Tests binary ops where one of the operands is a constant"""
interface_api = "c"
use_unpacked_api = True
test_runner = AOT_USMP_CORSTONE300_RUNNER
dtype = "int8"
shape = [1, 16, 16, 3]
input_0_scale = 0.256
input_0_zero_point = 33
input_1_scale = 0.128
input_1_zero_point = -24
model = make_model(
op,
input_0,
input_1,
input_0_scale,
input_0_zero_point,
input_1_scale,
input_1_zero_point,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
# validate pattern matching
assert_partitioned_function(orig_mod, cmsisnn_mod)
# validate the output
in_min, in_max = get_range_for_dtype_str(dtype)
inputs = {}
if isinstance(input_0, tvm.relay.expr.Var):
inputs.update({
"input_0":
np.random.randint(in_min, high=in_max, size=shape, dtype=dtype)
})
if isinstance(input_1, tvm.relay.expr.Var):
inputs.update({
"input_1":
np.random.randint(in_min, high=in_max, size=shape, dtype=dtype)
})
output_list = generate_ref_data(orig_mod["main"], inputs)
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=inputs,
outputs=output_list,
output_tolerance=1,
),
test_runner,
interface_api,
use_unpacked_api,
)
def create_aot_session(
platform,
board,
target,
mod,
params,
build_dir=Path(tempfile.mkdtemp()),
tune_logs=None,
timeout_override=None,
use_cmsis_nn=False,
project_options=None,
):
"""AOT-compiles and uploads a model to a microcontroller, and returns the RPC session"""
executor = tvm.relay.backend.Executor("aot")
crt_runtime = tvm.relay.backend.Runtime("crt", {"system-lib": True})
with ExitStack() as stack:
config = {"tir.disable_vectorize": True}
if use_cmsis_nn:
config["relay.ext.cmsisnn.options"] = {"mcpu": target.mcpu}
stack.enter_context(tvm.transform.PassContext(opt_level=3, config=config))
if use_cmsis_nn:
mod = cmsisnn.partition_for_cmsisnn(mod, params, mcpu=target.mcpu)
if tune_logs is not None:
stack.enter_context(tvm.autotvm.apply_history_best(tune_logs))
lowered = tvm.relay.build(
mod,
target=target,
params=params,
runtime=crt_runtime,
executor=executor,
)
parameter_size = len(tvm.runtime.save_param_dict(lowered.get_params()))
print(f"Model parameter size: {parameter_size}")
project = tvm.micro.generate_project(
str(tvm.micro.get_microtvm_template_projects(platform)),
lowered,
build_dir / "project",
{
f"{platform}_board": board,
"project_type": "host_driven",
# {} shouldn't be the default value for project options ({}
# is mutable), so we use this workaround
**(project_options or {}),
},
)
project.build()
project.flash()
return tvm.micro.Session(project.transport(), timeout_override=timeout_override)
def test_int8_pool_with_float32_input(
pool_size,
strides,
padding,
relu_type,
):
"""Tests QNN maxpool partitions with float32 input"""
interface_api = "c"
use_unpacked_api = True
test_runner = AOT_USMP_CORSTONE300_RUNNER
in_shape = (1, 28, 28, 12)
zero_point, scale = (-34, 0.0256)
input_ = relay.var("input", shape=in_shape, dtype="float32")
op = relay.op.add(input_, input_)
op = relay.qnn.op.quantize(op, relay.const(scale), relay.const(zero_point),
-1, "int8")
model = make_model(
pool_op=relay.nn.max_pool2d,
shape=in_shape,
pool_size=pool_size,
strides=strides,
padding=padding,
scale=scale,
zero_point=zero_point,
relu_type=relu_type,
input_op=op,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
# validate pattern matching
assert_partitioned_function(orig_mod, cmsisnn_mod)
# validate the output
np.random.seed(0)
inputs = {"input": np.random.uniform(0, 1, in_shape).astype("float32")}
output_list = generate_ref_data(orig_mod["main"], inputs)
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=inputs,
outputs=output_list,
params=None,
output_tolerance=1,
),
test_runner,
interface_api,
use_unpacked_api,
)
def test_invalid_softmax(in_dtype, out_dtype, zero_point, scale, out_zero_point, out_scale):
model = make_model(
[1, 16, 16, 3], in_dtype, out_dtype, zero_point, scale, out_zero_point, out_scale
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
attrs = [
cmsisnn_mod[var.name_hint].attrs
for var in cmsisnn_mod.get_global_vars()
if cmsisnn_mod[var.name_hint].attrs
]
assert not any(attrs), "No function should have an external attribute."
def test_op_int8(op, relu_type, input_0_scale, input_0_zero_point,
input_1_scale, input_1_zero_point):
interface_api = "c"
use_unpacked_api = True
test_runner = AOT_USMP_CORSTONE300_RUNNER
dtype = "int8"
shape = [1, 16, 16, 3]
model = make_model(
op,
generate_variable("input_0"),
generate_variable("input_1"),
input_0_scale,
input_0_zero_point,
input_1_scale,
input_1_zero_point,
relu_type,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
# validate pattern matching
assert_partitioned_function(orig_mod, cmsisnn_mod)
# validate the output
in_min, in_max = get_range_for_dtype_str(dtype)
inputs = {
"input_0": np.random.randint(in_min,
high=in_max,
size=shape,
dtype=dtype),
"input_1": np.random.randint(in_min,
high=in_max,
size=shape,
dtype=dtype),
}
output_list = generate_ref_data(orig_mod["main"], inputs)
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=inputs,
outputs=output_list,
output_tolerance=1,
),
test_runner,
interface_api,
use_unpacked_api,
)
def test_invalid_parameters(
in_dtype,
kernel_dtype,
kernel_zero_point,
):
"""Tests Depthwise op for non int8 inputs"""
ifm_shape = (1, 28, 28, 12)
out_channels = 2
input_scale = 1
input_zero_point = 24
kernel_scale = [0.11, 0.0237]
kernel_layout = "HWIO"
kernel_shape = [3, 3, ifm_shape[3], out_channels]
output_scale, output_zero_point = get_conv2d_qnn_params(
kernel_shape,
input_scale,
input_zero_point,
kernel_scale,
kernel_zero_point,
in_dtype,
kernel_dtype,
in_dtype,
False,
)
model, params = make_model(
shape=ifm_shape,
kernel_shape=kernel_shape,
input_zero_point=input_zero_point,
input_scale=input_scale,
kernel_zero_point=kernel_zero_point,
kernel_scale=kernel_scale,
output_zero_point=output_zero_point,
output_scale=output_scale,
padding="SAME",
strides=(1, 1),
dilation=(1, 1),
groups=1,
dtype=in_dtype,
kernel_dtype=kernel_dtype,
out_channels=out_channels,
weight_format=kernel_layout,
enable_bias=True,
relu_type="NONE",
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod, params)
assert_no_external_function(cmsisnn_mod)
def test_invalid_parameters(
in_dtype,
kernel_dtype,
kernel_zero_point,
):
in_shape = (2, 28)
out_channels = 2
input_scale = 1
input_zero_point = 24
kernel_scale = [0.11, 0.0237]
in_min, in_max = get_range_for_dtype_str(in_dtype)
kernel_shape = [out_channels, in_shape[1]]
conv2d_kernel_shape = [1, 1, kernel_shape[0], kernel_shape[1]]
output_scale, output_zero_point = get_conv2d_qnn_params(
conv2d_kernel_shape,
input_scale,
input_zero_point,
kernel_scale,
kernel_zero_point,
in_dtype,
kernel_dtype,
in_dtype,
)
model, params = make_model(
in_shape=in_shape,
kernel_shape=kernel_shape,
input_zero_point=input_zero_point,
kernel_zero_point=kernel_zero_point,
input_scale=input_scale,
kernel_scale=kernel_scale,
output_zero_point=output_zero_point,
output_scale=output_scale,
dtype=in_dtype,
kernel_dtype=kernel_dtype,
out_channels=out_channels,
enable_bias=True,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod, params)
# validate pattern matching
attrs = [
cmsisnn_mod[var.name_hint].attrs
for var in cmsisnn_mod.get_global_vars()
if cmsisnn_mod[var.name_hint].attrs
]
assert not any(attrs), "No function should have an external attribute."
def test_invalid_parameters():
model = make_model(
pool_op=relay.nn.avg_pool2d,
shape=(1, 28, 28, 12),
pool_size=(1, 1),
strides=(1, 1),
padding="VALID",
dtype="uint8",
scale=1,
zero_point=-33,
relu_type="RELU",
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
assert_no_external_function(cmsisnn_mod)
def test_workspace_calculation_cmsis_nn():
"""This tests cmsis_nn codegen for workspace calculation.
This is tested specially because cmsis-nn codegen creates
multiple PrimFuncs per offloaded relay function in a non
-hierarchical manner."""
pytest.importorskip("tflite")
# pylint: disable=import-outside-toplevel
from tvm.relay.op.contrib import cmsisnn
from tvm.contrib.download import download_testdata
# pylint: enable=import-outside-toplevel
target = "c"
runtime = Runtime("crt")
executor = Executor(
"aot",
{
"workspace-byte-alignment": 16,
"interface-api": "c",
"unpacked-api": True,
},
)
base_url = ("https://github.com/ARM-software/ML-zoo/raw/"
"48a22ee22325d15d2371a6df24eb7d67e21dcc97"
"/models/keyword_spotting/cnn_small/tflite_int8")
file_to_download = "cnn_s_quantized.tflite"
file_saved = "cnn_s_quantized_15Dec2021.tflite"
model_file = download_testdata("{}/{}".format(base_url, file_to_download),
file_saved)
mod, _, params = create_relay_module_and_inputs_from_tflite_file(
model_file)
mod = cmsisnn.partition_for_cmsisnn(mod, params)
with tvm.transform.PassContext(
opt_level=3,
config={
"tir.disable_vectorize": True,
},
):
lib = tvm.relay.build(mod,
target,
executor=executor,
runtime=runtime,
params=params)
mlf_memory_map = mlf._build_function_memory_map(lib.function_metadata)
assert mlf_memory_map["main"][0]["workspace_size_bytes"] == 14384
def test_both_scalar_inputs_int8(op, ):
input_scale = 0.256
input_zero_point = 33
dtype = "int8"
model = make_model(
op,
generate_scalar_constant(),
generate_scalar_constant(),
input_scale,
input_zero_point,
input_scale,
input_zero_point,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
assert_no_external_function(cmsisnn_mod)
def test_both_scalar_inputs_int8(op, ):
"""Tests binary ops where both operands are scalars"""
input_scale = 0.256
input_zero_point = 33
model = make_model(
op,
generate_scalar_constant(),
generate_scalar_constant(),
input_scale,
input_zero_point,
input_scale,
input_zero_point,
)
orig_mod = make_module(model)
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod)
assert_no_external_function(cmsisnn_mod)
def test_cnn_small(test_runner):
"""Download a small network and tests TVM via CMSIS-NN output against TFLite output"""
# download the model
base_url = ("https://github.com/ARM-software/ML-zoo/raw/"
"48a22ee22325d15d2371a6df24eb7d67e21dcc97"
"/models/keyword_spotting/cnn_small/tflite_int8")
file_to_download = "cnn_s_quantized.tflite"
file_saved = "cnn_s_quantized_15Dec2021.tflite"
model_file = download_testdata("{}/{}".format(base_url, file_to_download),
file_saved)
with open(model_file, "rb") as f:
tflite_model_buf = f.read()
input_shape = (1, 490)
dtype = "int8"
in_min, in_max = get_range_for_dtype_str(dtype)
rng = np.random.default_rng(12345)
input_data = rng.integers(in_min,
high=in_max,
size=input_shape,
dtype=dtype)
orig_mod, params = _convert_to_relay(tflite_model_buf, input_data, "input")
cmsisnn_mod = cmsisnn.partition_for_cmsisnn(orig_mod, params)
# validate CMSIS-NN output against CPU output
interface_api = "c"
use_unpacked_api = True
inputs = {"input": input_data}
params = {}
output_list = generate_ref_data(orig_mod["main"], inputs, params)
compile_and_run(
AOTTestModel(
module=cmsisnn_mod,
inputs=inputs,
outputs=output_list,
params=params,
output_tolerance=1,
),
test_runner,
interface_api,
use_unpacked_api,
)
