def test_split():
if not ethosn_available():
return
trials = [
((1, 16, 16, 32), (2, 7, 10), 2),
((1, 12, 8, 16), 3, 1),
((1, 33), 11, 1),
]
np.random.seed(0)
for shape, splits, axis in trials:
outputs = []
inputs = {
"a":
tvm.nd.array(
np.random.randint(0, high=256, size=shape, dtype="uint8"))
}
for npu in [False, True]:
model = _get_model(shape, "uint8", splits, axis)
mod = tei.make_module(model, {})
output_count = splits if type(splits) == int else len(splits) + 1
outputs.append(
tei.build_and_run(mod, inputs, output_count, {}, npu=npu))
tei.verify(outputs, 0)
def test_output_order():
if not ethosn_available():
return
def get_model(input_shape, var_names):
"""Return a model"""
a = relay.var(next(var_names), shape=input_shape, dtype="uint8")
z = relay.op.clip(a, 0, 255)
b = relay.op.clip(z, 0, 15)
c = relay.op.clip(z, 16, 31)
d = relay.op.clip(z, 32, 47)
e = relay.op.clip(z, 48, 63)
f = relay.op.clip(z, 64, 79)
g = relay.op.clip(z, 80, 95)
h = relay.op.clip(z, 96, 111)
i = relay.op.clip(z, 112, 127)
return relay.Tuple((d, c, e, f, i, b, h, g))
inputs = {
"a":
tvm.nd.array(
np.random.randint(0, high=255, size=(1, 16, 16, 4),
dtype="uint8")),
}
outputs = []
for npu in [False, True]:
model = get_model(inputs["a"].shape, iter(inputs))
mod = tei.make_module(model, [])
outputs.append(tei.build_and_run(mod, inputs, 8, {}, npu=npu))
tei.verify(outputs, 1)
def test_sigmoid_failure():
if not ethosn_available():
return
trials = [
((2, 4, 4, 4), 64, 0.2, 0, 1 / 256, "uint8", "batch size=2, batch size must = 1"),
(
(1, 4, 4, 4),
64,
0.2,
0,
1 / 256,
"int8",
"dtype='int8', dtype must be either uint8 or int32",
),
(
(1, 4, 4, 4),
64,
0.2,
0,
1,
"uint8",
"output quantization params=(0, 1), must = (0, 1/256)",
),
]
for shape, input_zp, input_sc, output_zp, output_sc, dtype, err_msg in trials:
model = _get_model(shape, input_zp, input_sc, output_zp, output_sc, dtype)
model = tei.make_ethosn_composite(model, "ethos-n.qnn_sigmoid")
mod = tei.make_ethosn_partition(model)
tei.test_error(mod, {}, err_msg)
def test_fullyconnected():
if not ethosn_available():
return
trials = [
((1, 1024), 71, 0.580, 79, 1.498),
((1, 4096), 166, 1.724, 117, 0.180),
((1, 16384), 101, 1.372, 21, 1.346),
]
np.random.seed(0)
for shape, input_zp, input_sc, kernel_zp, kernel_sc in trials:
inputs = {
"a":
tvm.nd.array(
np.random.randint(0, high=255, size=shape, dtype="uint8")),
}
outputs = []
output_zp, output_sc = tei.get_conv2d_qnn_params(
input_zp, input_sc, kernel_zp, kernel_sc, shape[0], shape[1], 1)
for npu in [False, True]:
model, params = _get_model(
shape,
shape,
input_zp,
input_sc, # input zp, sc
kernel_zp,
kernel_sc, # kernel
output_zp,
output_sc, # output
"uint8",
)
mod = tei.make_module(model, params)
outputs.append(tei.build_and_run(mod, inputs, 1, params, npu=npu))
tei.verify(outputs, 1)
def test_pooling():
if not ethosn_available():
return
trials = [
((1, 8, 8, 8), relay.nn.max_pool2d, (2, 2), (2, 2), (0, 0, 0, 0),
"NHWC"),
((1, 9, 9, 9), relay.nn.max_pool2d, (2, 2), (2, 2), (0, 0, 1, 1),
"NHWC"),
((1, 9, 9, 9), relay.nn.max_pool2d, (3, 3), (2, 2), (0, 0, 0, 0),
"NHWC"),
((1, 8, 8, 8), relay.nn.max_pool2d, (3, 3), (2, 2), (0, 0, 1, 1),
"NHWC"),
((1, 8, 8, 8), relay.nn.avg_pool2d, (3, 3), (1, 1), (1, 1, 1, 1),
"NHWC"),
]
for shape, typef, size, stride, pad, layout in trials:
inputs = {
"a":
tvm.nd.array(
np.random.randint(low=0, high=255, size=shape, dtype="uint8")),
}
outputs = []
model = _get_model(shape, typef, size, stride, pad, layout, "uint8")
for npu in [False, True]:
mod = tei.make_module(model, {})
outputs.append(tei.build_and_run(mod, inputs, 1, {}, npu=npu))
tei.verify(outputs, 1)
def test_conv2d_failure():
if not ethosn_available():
return
trials = [
((1, 4, 4, 4), 1, 1, 0, 1, 0, 1, 0, 1, "none", (1, 1), (1, 1), 1,
"uint8", 8, "HWIO",
"Overall scale (of the input * weights / output) should be in the range [0, 1)"
),
((1, 4, 4,
4), 1, 1, 0, 1, 0, 1, 0, 1, "none", (1, 1), (1, 1), 1, "int8", 8,
"HWIO", "dtype='int8', dtype must be either uint8 or int32"),
((1, 4, 4, 4), 2, 2, 0, 1, 0, 1, 0, 2, "both", (1, 1), (1, 1), 1,
"uint8", 8, "HWIO",
"both op and attr padding exist, must be either op/attr only or no padding"
),
((1, 4, 4, 4), 1, 1, 0, 1, 0, 1, 0, 2, "none", (1, 1, 1), (1, 1), 1,
"uint8", 8, "HWIO", "stride size=3, stride size must = 2"),
((1, 4, 4, 4), 1, 1, 0, 1, 0, 1, 0, 2, "none", (1, 1), (2, 1), 1,
"uint8", 8, "HWIO", "dilation=[2, 1], dilation must = [1, 1]"),
((2, 4, 4, 4), 1, 1, 0, 1, 0, 1, 0, 2, "none", (1, 1), (1, 1), 1,
"uint8", 8, "HWIO", "batch size=2, batch size must = 1"),
]
np.random.seed(0)
for shape, kernel_h, kernel_w, input_zp, input_sc, kernel_zp,\
kernel_sc, output_zp, output_sc, pad, stride, dilation,\
groups, dtype, out_channels, weight_format, err_msg in trials:
model, params = _get_model(shape, kernel_h, kernel_w, input_zp,
input_sc, kernel_zp, kernel_sc, output_zp,
output_sc, pad, stride, dilation, groups,
dtype, out_channels, weight_format)
model = tei.make_ethosn_composite(model, "ethos-n.qnn_conv2d")
mod = tei.make_ethosn_partition(model)
tei.test_error(mod, {}, err_msg)
def test_addition_failure():
if not ethosn_available():
return
trials = [
(
(2, 4, 4, 4),
"uint8",
0,
1,
0,
1,
0,
1,
"batch size=2, batch size must = 1; batch size=2, batch size must = 1",
),
(
(1, 4, 4, 4),
"int8",
0,
1,
0,
1,
0,
1,
"dtype='int8', dtype must be either uint8 or int32; dtype='int8', dtype must be either uint8 or int32",
),
]
for shape, dtype, lhs_zp, lhs_sc, rhs_zp, rhs_sc, out_zp, out_sc, err_msg in trials:
model = _get_model(shape, lhs_zp, lhs_sc, rhs_zp, rhs_sc, out_zp,
out_sc, dtype)
mod = tei.make_ethosn_partition(model)
tei.test_error(mod, {}, err_msg)
def test_addition():
if not ethosn_available():
return
trials = [
((1, 22, 9, 9), 24, 1.057, 253, 0.452),
((1, 27, 21, 16), 79, 0.850, 24, 0.380),
((1, 7, 12, 28), 125, 1.293, 239, 0.320),
((1, 14, 9, 6), 14, 0.942, 227, 1.562),
((1, 13, 16, 22), 15, 0.727, 180, 0.461),
]
np.random.seed(0)
for shape, rhs_zp, rhs_sc, lhs_zp, lhs_sc in trials:
outputs = []
inputs = {
"a":
tvm.nd.array(
np.random.randint(0, high=255, size=shape, dtype="uint8")),
"b":
tvm.nd.array(
np.random.randint(0, high=255, size=shape, dtype="uint8")),
}
out_zp, out_sc = _get_addition_qnn_params(lhs_zp, lhs_sc, rhs_zp,
rhs_sc)
model = _get_model(shape, lhs_zp, lhs_sc, rhs_zp, rhs_sc, out_zp,
out_sc, "uint8")
for npu in [False, True]:
mod = tei.make_module(model, [])
outputs.append(tei.build_and_run(mod, inputs, 1, {}, npu=npu))
tei.verify(outputs, 2)
def test_conv2d():
if not ethosn_available():
return
trials = [
[(1, 17, 20, 26), 4, 3, 1, 'attr', (2, 2), (1, 1)],
[(1, 30, 27, 30), 5, 5, 3, 'none', (1, 1), (1, 1)],
[(1, 14, 28, 11), 6, 2, 2, 'op', (2, 2), (1, 1)],
[(1, 9, 20, 30), 7, 1, 5, 'none', (1, 1), (1, 1)],
[(1, 21, 21, 22), 8, 5, 1, 'attr', (2, 2), (1, 1)],
[(1, 21, 25, 29), 9, 2, 5, 'op', (1, 1), (1, 1)],
[(1, 31, 28, 15), 10, 1, 2, 'attr', (2, 2), (1, 1)],
[(1, 21, 21, 8), 11, 3, 3, 'none', (1, 1), (1, 1)],
[(1, 5, 11, 6), 12, 5, 2, 'op', (2, 2), (1, 1)],
[(1, 12, 7, 18), 13, 1, 3, 'op', (1, 1), (1, 1)],
[(1, 24, 6, 26), 14, 3, 5, 'none', (2, 2), (1, 1)],
[(1, 19, 24, 16), 15, 2, 1, 'attr', (1, 1), (1, 1)],
]
np.random.seed(0)
for depthwise in [False, True]:
for shape, out_channels, kernel_h, kernel_w, pad, stride, dilation in trials:
if depthwise:
out_channels = shape[3]
groups = out_channels
kernel_w = kernel_h
weight_format = "HWOI"
stride = (1, 1) if kernel_w == 1 else (2, 2)
else:
groups = 1
weight_format = "HWIO"
outputs = []
inputs = {
"a":
tvm.nd.array(
np.random.randint(0, high=255, size=shape, dtype="uint8")),
}
input_zp = np.random.randint(0, 255)
input_sc = np.random.random() * 2
kernel_zp = np.random.randint(0, 255)
kernel_sc = np.random.random() * 2
output_zp, output_sc = _get_conv2d_qnn_params(
input_zp, input_sc, kernel_zp, kernel_sc, kernel_h, kernel_w,
shape[3])
model, params = _get_model(shape, kernel_h, kernel_w, input_zp,
input_sc, kernel_zp, kernel_sc,
output_zp, output_sc, pad, stride,
dilation, groups, "uint8", out_channels,
weight_format)
for npu in [False, True]:
mod = tei.make_module(model, params)
outputs.append(
tei.build_and_run(mod, inputs, 1, params, npu=npu))
tei.verify(outputs, 1)
def test_constant_duplication():
if not ethosn_available():
return
model, params = _get_model()
mod = tei.make_module(model, params)
res = tei.build(mod, params, npu=True, expected_host_ops=1)
for key, value in res.params.items():
assert key == "p0"
assert value.numpy().size == 64
def test_split_add_concat():
if not ethosn_available():
return
def get_model(input_shape, var_names):
"""Return a model"""
a = relay.var(next(var_names), shape=input_shape, dtype="uint8")
split_scale = relay.const(0.25, "float32")
split_zp = relay.const(100, "int32")
add_scale = relay.const(0.75, "float32")
add_zp = relay.const(120, "int32")
axis = 2
split = relay.split(a, indices_or_sections=4, axis=axis)
b = relay.qnn.op.add(
split[0],
split[1],
lhs_scale=split_scale,
lhs_zero_point=split_zp,
rhs_scale=split_scale,
rhs_zero_point=split_zp,
output_scale=add_scale,
output_zero_point=add_zp,
)
conc = relay.qnn.op.concatenate(
[b, split[2], split[3]],
input_scales=(add_scale, split_scale, split_scale),
input_zero_points=(add_zp, split_zp, split_zp),
output_scale=add_scale,
output_zero_point=add_zp,
axis=axis,
)
return conc
inputs = {
"a":
tvm.nd.array(
np.random.randint(0, high=255, size=(1, 16, 16, 4),
dtype="uint8")),
}
outputs = []
for npu in [False, True]:
model = get_model(inputs["a"].shape, iter(inputs))
mod = tei.make_module(model, [])
outputs.append(tei.build_and_run(mod, inputs, 1, {}, npu=npu))
tei.verify(outputs, 2)
def _test_image_network(
model_url,
model_sub_path,
input_dict,
compile_hash,
output_count,
run=True,
host_ops=0,
npu_partitions=1,
):
if not ethosn_available():
return
def get_model():
if model_url[-3:] in ("tgz", "zip"):
model_path = tf_testing.get_workload_official(
model_url,
model_sub_path,
)
else:
model_path = download.download_testdata(
model_url,
model_sub_path,
)
return _get_tflite_model(model_path, input_dict, "uint8")
outputs = []
inputs = {}
for input_name in input_dict:
input_shape = input_dict[input_name]
inputs[input_name] = tei.get_real_image(input_shape[1], input_shape[2])
for npu in [False, True]:
mod, params = get_model()
graph, lib, params = tei.build(mod,
params,
npu=npu,
expected_host_ops=host_ops,
npu_partitions=npu_partitions)
if npu:
tei.assert_lib_hash(lib, compile_hash)
if run:
outputs.append(
tei.run(graph, lib, params, inputs, output_count, npu=npu))
if run:
tei.verify(outputs, 1, verify_saturation=False)
def test_multiple_command_streams(dtype):
"""Check that multiple Ethos-N partitions are correctly handled.
If there's more than one Ethos-N graph partition, more than one command
stream will be created. This should be handled correctly by both the
Ethos-N codegen and Ethos-N runtime module. This test checks against a
simple graph which creates two Ethos-N partitions and checks the result
against an 'all-CPU' run through TVM.
"""
def get_model(dtype):
"""
max_pool2d
|
abs
|
max_pool2d
"""
x = relay.var("x", shape=(1, 4, 4, 4), dtype=dtype)
out = relay.nn.max_pool2d(x, (2, 2), (2, 2),
layout="NHWC") # supported
out = relay.op.abs(out) # not supported
out = relay.nn.max_pool2d(out, (2, 2), (2, 2),
layout="NHWC") # supported
return out
np.random.seed(0)
inputs = {
"x":
tvm.nd.array(
np.random.randint(np.iinfo(dtype).min,
np.iinfo(dtype).max + 1,
size=(1, 4, 4, 4),
dtype=dtype))
}
model = get_model(dtype)
mod = tei.make_module(model, {})
# Mock inference is only supported when the whole graph is offloaded to the NPU
if ethosn_available() == Available.SW_ONLY:
tei.build(mod, {}, npu=True, expected_host_ops=1, npu_partitions=2)
else:
tei.build_and_run(mod,
inputs,
1, {},
npu=True,
expected_host_ops=1,
npu_partitions=2)
def test_multiple_command_streams():
"""Check that multiple Ethos-N partitions are correctly handled.
If there's more than one Ethos-N graph partition, more than one command
stream will be created. This should be handled correctly by both the
Ethos-N codegen and Ethos-N runtime module. This test checks against a
simple graph which creates two Ethos-N partitions and checks the result
against an 'all-CPU' run through TVM.
"""
if ethosn_available() != Available.SW_AND_HW:
return
def get_model():
"""
max_pool2d
|
abs
|
max_pool2d
"""
x = relay.var("x", shape=(1, 4, 4, 4), dtype="uint8")
out = relay.nn.max_pool2d(x, (2, 2), (2, 2),
layout="NHWC") # supported
out = relay.op.abs(out) # not supported
out = relay.nn.max_pool2d(out, (2, 2), (2, 2),
layout="NHWC") # supported
return out
np.random.seed(0)
outputs = []
inputs = {
"x":
tvm.nd.array(
np.random.randint(0, high=256, size=(1, 4, 4, 4), dtype="uint8"))
}
for npu in [False, True]:
model = get_model()
mod = tei.make_module(model, {})
outputs.append(
tei.build_and_run(mod,
inputs,
1, {},
npu=npu,
expected_host_ops=1,
npu_partitions=2))
tei.verify(outputs, 0)
def test_relu_failure():
if not ethosn_available():
return
trials = [
((1, 4, 4, 4, 4), "uint8", 65, 78,
"dimensions=5, dimensions must be <= 4"),
((1, 8, 4, 2), "int8", 1, 254,
"dtype='int8', dtype must be either uint8 or int32"),
((1, 8, 4, 2), "uint8", 254, 1, "Relu has lower bound > upper bound"),
((2, 2, 2, 2), "uint8", 1, 63, "batch size=2, batch size must = 1; "),
]
for shape, dtype, a_min, a_max, err_msg in trials:
model = _get_model(shape, dtype, a_min, a_max)
mod = tei.make_ethosn_partition(model)
tei.test_error(mod, {}, err_msg)
def test_split_with_asym_concats():
if not ethosn_available():
return
def get_model(shape, splits, axis):
a = relay.var("a", shape=shape, dtype="uint8")
split = relay.op.split(a, indices_or_sections=splits, axis=axis)
zeroi = relay.const(1, "int32")
zerof = relay.const(0.5, "float32")
con1 = relay.qnn.op.concatenate(
[split[0], split[1]],
input_scales=[zerof] * 2,
input_zero_points=[zeroi] * 2,
output_scale=zerof,
output_zero_point=zeroi,
axis=axis,
)
con2 = relay.qnn.op.concatenate(
[split[2], split[3]],
input_scales=[zerof] * 2,
input_zero_points=[zeroi] * 2,
output_scale=zerof,
output_zero_point=zeroi,
axis=axis,
)
return relay.Tuple((con2, con1))
trials = [
((1, 16, 16, 32), (2, 7, 10), 2),
]
np.random.seed(0)
for shape, splits, axis in trials:
outputs = []
inputs = {
"a":
tvm.nd.array(
np.random.randint(0, high=256, size=shape, dtype="uint8"))
}
for npu in [False, True]:
model = get_model(shape, splits, axis)
mod = tei.make_module(model, {})
outputs.append(tei.build_and_run(mod, inputs, 2, {}, npu=npu))
tei.verify(outputs, 0)
def requires_ethosn(*args):
"""Mark a test as requiring Arm(R) Ethos(TM)-N to run.
Parameters
----------
f : function
Function to mark
"""
marks = [
pytest.mark.ethosn,
pytest.mark.skipif(
not ethosn_available(),
reason=("Arm(R) Ethos(TM)-N support not enabled. "
"Set USE_ETHOSN=ON in config.cmake to enable, "
"and ensure that hardware support is present."),
),
]
return _compose(args, marks)
def test_mobilenet_v1():
# If this test is failing due to a hash mismatch, please notify @mbaret and
# @Leo-arm. The hash is there to catch any changes in the behaviour of the
# codegen, which could come about from either a change in Support Library
# version or a change in the Ethos-N codegen. To update this requires running
# on hardware that isn't available in CI.
hw = ethosn_available()
_test_image_network(
model_url="https://storage.googleapis.com/download.tensorflow.org/"
"models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz",
model_sub_path="mobilenet_v1_1.0_224_quant.tflite",
input_dict={"input": (1, 224, 224, 3)},
compile_hash="81637c89339201a07dc96e3b5dbf836a",
output_count=1,
run=(hw == Available.SW_AND_HW),
host_ops=3,
npu_partitions=1,
)
def test_depth_to_space_failure():
if not ethosn_available():
return
trials = [
((2, 16, 16, 16), 2, "uint8", "NHWC",
"batch size=2, batch size must = 1"),
((1, 16, 16, 16), 2, "int8", "NHWC",
"dtype='int8', dtype must be either uint8 or int32"),
((1, 16, 16, 16), 4, "uint8", "NHWC",
"Only block size of 2 is supported"),
((1, 16, 16, 16), 2, "uint8", "NCHW",
"layout=NCHW, layout must = NHWC"),
]
for shape, block, dtype, layout, err_msg in trials:
model = _get_model(shape, block, dtype, layout)
mod = tei.make_ethosn_partition(model)
tei.test_error(mod, {}, err_msg)
def test_input_tuples():
if not ethosn_available():
return
def get_model(shapes, axis):
tup = []
for i, shape in enumerate(shapes):
a = relay.var("in" + str(i), shape=shape, dtype="uint8")
tup.append(a)
zeroi = relay.const(1, "int32")
zerof = relay.const(0.5, "float32")
con = relay.qnn.op.concatenate(
tup,
input_scales=[zerof] * len(shapes),
input_zero_points=[zeroi] * len(shapes),
output_scale=zerof,
output_zero_point=zeroi,
axis=axis,
)
return con
np.random.seed(0)
inputs = {
"in0":
tvm.nd.array(np.random.randint(0, high=256, size=(1, 4),
dtype="uint8")),
"in1":
tvm.nd.array(np.random.randint(0, high=256, size=(1, 6),
dtype="uint8")),
}
outputs = []
for npu in [False, True]:
model = get_model([(1, 4), (1, 6)], 1)
if not npu:
mod = tei.make_module(model, {})
else:
mod = tei.make_ethosn_partition(model)
lib = tei.build(mod, {}, npu=False)
outputs.append(tei.run(lib, inputs, 1, npu=npu))
tei.verify(outputs, 0)
def test_concatenate():
if not ethosn_available():
return
trials = [
([(1, 4), (1, 6)], 1),
([(1, 16, 4), (1, 16, 4)], 1),
([(1, 25, 4, 16)] * 3, 3),
([(1, 25, 4, 16), (1, 25, 5, 16), (1, 25, 6, 16)], 2),
]
for shapes, axis in trials:
outputs = []
inputs = _get_inputs(shapes)
for npu in [False, True]:
model = _get_model(shapes, "uint8", axis)
mod = tei.make_module(model, {})
outputs.append(tei.build_and_run(mod, inputs, 1, {}, npu=npu))
tei.verify(outputs, 0)
def test_split_failure():
if not ethosn_available():
return
trials = [
((1, 4, 4, 4, 4), "uint8", 4, 2,
"dimensions=5, dimensions must be <= 4;"),
((1, 4, 4, 4), "int8", 4, 2,
"dtype='int8', dtype must be either uint8 or int32;"),
((2, 4, 4, 4), "uint8", 4, 2, "batch size=2, batch size must = 1;"),
((1, 4, 4, 4), "uint8", 1, 0,
"Split cannot be performed along batch axis (axis 0);"),
((1, 4, 4, 4), "uint8", 4, 3,
"Split along the channels dimension (axis 3) requires all output sizes (specified in splitInfo.m_Sizes) to be multiples of 16;"
),
]
for shape, dtype, splits, axis, err_msg in trials:
model = _get_model(shape, dtype, splits, axis)
mod = tei.make_ethosn_partition(model)
tei.test_error(mod, {}, err_msg)
def test_sigmoid():
if not ethosn_available():
return
trials = [
(1, 16, 16, 16),
(1, 8, 8),
]
np.random.seed(0)
for shape in trials:
inputs = {
"a": tvm.nd.array(np.random.randint(0, high=255, size=shape, dtype="uint8")),
}
outputs = []
for npu in [False, True]:
model = _get_model(shape, 64, 0.02, 0, 1 / 256, "uint8")
mod = tei.make_module(model, [])
outputs.append(tei.build_and_run(mod, inputs, 1, {}, npu=npu))
tei.verify(outputs, 1)
def test_depth_to_space():
if not ethosn_available():
return
trials = [
(1, 16, 16, 16),
(1, 64, 32, 16),
]
for shape in trials:
inputs = {
"a":
tvm.nd.array(
np.random.randint(0, high=255, size=shape, dtype="uint8")),
}
outputs = []
for npu in [False, True]:
model = _get_model(shape, 2, "uint8", "NHWC")
mod = tei.make_module(model, {})
outputs.append(tei.build_and_run(mod, inputs, 1, {}, npu=npu))
tei.verify(outputs, 1)
def test_relu():
if not ethosn_available():
return
trials = [
((1, 4, 4, 4), 65, 178),
((1, 8, 4, 2), 1, 254),
((1, 16), 12, 76),
]
for shape, a_min, a_max in trials:
inputs = {
"a":
tvm.nd.array(
np.random.randint(0, high=255, size=shape, dtype="uint8")),
}
outputs = []
for npu in [False, True]:
model = _get_model(inputs["a"].shape, "uint8", a_min, a_max)
mod = tei.make_module(model, {})
outputs.append(tei.build_and_run(mod, inputs, 1, {}, npu=npu))
tei.verify(outputs, 1)
def test_output_tuple_propagation():
"""This tests the case where the output tuple must be inferred
as having dummy tensor information."""
if not ethosn_available():
return
def get_model():
a = relay.var("a", shape=(1, 4, 4, 16), dtype="uint8")
split = relay.op.split(a, indices_or_sections=4, axis=2)
return relay.Tuple((split[0], split[1], split[2], split[3]))
np.random.seed(0)
outputs = []
inputs = {
"a":
tvm.nd.array(
np.random.randint(0, high=256, size=(1, 4, 4, 16), dtype="uint8"))
}
for npu in [False, True]:
model = get_model()
mod = tei.make_module(model, {})
outputs.append(tei.build_and_run(mod, inputs, 4, {}, npu=npu))
tei.verify(outputs, 0)
def test_concatenate_failure():
if not ethosn_available():
return
trials = [
([(1, 4, 4, 4, 4), (1, 4, 4, 4, 4)], "uint8", 1,
"dimensions=5, dimensions must be <= 4;"),
([(1, 4, 4, 4), (1, 4, 4, 4)], "uint8", 3,
"Concatenation along the channels dimension (axis 3) requires input tensors with a multiple of 16 channels;"
),
([(1, 4, 4, 4), (1, 4, 4, 4)], "int8", 2,
"dtype='int8', dtype must be either uint8 or int32; dtype='int8', dtype must be either uint8 or int32;"
),
([(2, 4, 4, 4), (2, 4, 4, 4)], "uint8", 2,
"batch size=2, batch size must = 1; batch size=2, batch size must = 1;"
),
([(1, 4, 4, 4), (1, 4, 4, 4)], "uint8", 0,
"Concatenation cannot be performed along batch axis (axis 0);"),
]
for shapes, dtype, axis, err_msg in trials:
model = _get_model(shapes, dtype, axis)
mod = tei.make_ethosn_partition(model)
tei.test_error(mod, {}, err_msg)
graph, lib, params, dumps = tvmc.compile(
tflite_mobilenet_v1_0_25_128,
target="opencl",
target_host="llvm",
alter_layout="NCHW",
)
# check for output types
assert type(graph) is str
assert type(lib) is tvm.runtime.module.Module
assert type(params) is dict
assert type(dumps) is dict
@pytest.mark.skipif(
not ethosn_available(),
reason=
"--target=ethos-n77 is not available. TVM built with 'USE_ETHOSN OFF'",
)
def test_compile_tflite_module_with_external_codegen(
tflite_mobilenet_v1_1_quant):
pytest.importorskip("tflite")
graph, lib, params, dumps = tvmc.compile(tflite_mobilenet_v1_1_quant,
target="ethos-n77, llvm",
dump_code="relay")
# check for output types
assert type(graph) is str
assert type(lib) is tvm.runtime.module.Module
assert type(params) is dict
def _test_image_network(
model_url,
model_sub_path,
input_dict,
compile_hash,
output_count,
host_ops=0,
npu_partitions=1,
run=False,
):
"""Test an image network.
Parameters
----------
model_url : str
The URL to the model.
model_sub_path : str
The name of the model file.
input_dict : dict
The input dict.
compile_hash : str, set
The compile hash(es) to check the compilation output against.
output_count : int
The expected number of outputs.
host_ops : int
The expected number of host operators.
npu_partitions : int
The expected number of Ethos-N partitions.
run : bool
Whether or not to try running the network. If hardware isn't
available, the run will still take place but with a mocked
inference function, so the results will be incorrect. This is
therefore just to test the runtime flow is working rather than
to check the correctness/accuracy.
"""
if not ethosn_available():
return
def get_model():
if model_url[-3:] in ("tgz", "zip"):
model_path = tf_testing.get_workload_official(
model_url,
model_sub_path,
)
else:
model_path = download.download_testdata(
model_url,
model_sub_path,
)
return _get_tflite_model(model_path, input_dict, "uint8")
inputs = {}
for input_name in input_dict:
input_shape = input_dict[input_name]
inputs[input_name] = tei.get_real_image(input_shape[1], input_shape[2])
mod, params = get_model()
m = tei.build(mod, params, npu=True, expected_host_ops=host_ops, npu_partitions=npu_partitions)
tei.assert_lib_hash(m.get_lib(), compile_hash)
if run:
tei.run(m, inputs, output_count, npu=True)
def test_pooling_failure():
if not ethosn_available():
return
trials = [
(
(2, 8, 8, 8),
relay.nn.max_pool2d,
(2, 2),
(2, 2),
(0, 0, 0, 0),
"NHWC",
"uint8",
"batch size=2, batch size must = 1",
),
(
(1, 8, 8, 8),
relay.nn.max_pool2d,
(2, 2),
(2, 2),
(0, 0, 0, 0),
"NHWC",
"int8",
"dtype='int8', dtype must be either uint8 or int32",
),
(
(1, 8, 8, 8),
relay.nn.max_pool2d,
(2, 2),
(2, 2),
(0, 0, 0, 0),
"NCHW",
"uint8",
"data format=NCHW, data format must = NHWC",
),
(
(1, 8, 8, 8),
relay.nn.max_pool2d,
(2, 2),
(2, 2, 2),
(0, 0, 0, 0),
"NHWC",
"uint8",
"stride size=3, stride size must = 2",
),
(
(1, 8, 8, 8),
relay.nn.max_pool2d,
(2, 2, 2),
(2, 2),
(0, 0, 0, 0),
"NHWC",
"uint8",
"dimensions=3, dimensions must = 2",
),
]
for shape, typef, size, stride, pad, layout, dtype, err_msg in trials:
model = _get_model(shape, typef, size, stride, pad, layout, dtype)
mod = tei.make_ethosn_partition(model)
tei.test_error(mod, {}, err_msg)