def test_spacetodepth():
n, c, h, w = shape = (1, 1, 4, 6)
input1 = np.random.rand(n, c, h, w).astype("float32")
blocksize = 2
inputs = [helper.make_tensor_value_info("input1", TensorProto.FLOAT, shape=shape)]
outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=(1, 4, 2, 3))]
nodes = [helper.make_node("SpaceToDepth", ["input1"], ["output"], block_size=blocksize)]
graph = helper.make_graph(nodes,
"spacetodepth_test",
inputs,
outputs)
spacetodepth_model = helper.make_model(graph)
bkd_rep = backend.prepare(spacetodepth_model)
output = bkd_rep.run([input1])
tmp = np.reshape(input1, [n, c,
h // blocksize, blocksize,
w // blocksize, blocksize])
tmp = np.transpose(tmp, [0, 3, 5, 1, 2, 4])
numpy_op = np.reshape(tmp, [n, c * (blocksize**2),
h // blocksize,
w // blocksize])
npt.assert_almost_equal(output[0], numpy_op)
def test_nested_graph(self): # type: () -> None
n1 = helper.make_node(
"Scale", ["X"], ["Y"], scale=2., name="n1")
n2 = helper.make_node(
"Scale", ["Y"], ["Z"], scale=3., name="n2")
graph = helper.make_graph(
[n1, n2],
"nested",
inputs=[
helper.make_tensor_value_info("X", TensorProto.FLOAT, [1, 2])
],
outputs=[
helper.make_tensor_value_info("Z", TensorProto.FLOAT, [1, 2])
]
)
i1 = helper.make_node(
"If", ["cond"], ["Z"], then_branch=graph, else_branch=graph)
graph = helper.make_graph(
[i1],
"test",
inputs=[
helper.make_tensor_value_info("cond", TensorProto.BOOL, [1])
],
outputs=[],
)
checker.check_graph(graph)
def test_small_model(self):
# Create one input
X = helper.make_tensor_value_info('IN', TensorProto.FLOAT, [2, 3])
# Create one output
Y = helper.make_tensor_value_info('OUT', TensorProto.FLOAT, [2, 3])
# Create a node
node_def = helper.make_node('Abs', ['IN'], ['OUT'])
# Create the model
graph_def = helper.make_graph([node_def], "test-model", [X], [Y])
onnx_model = helper.make_model(graph_def,
producer_name='onnx-example')
model = Model()
model.BuildFromOnnxModel(onnx_model)
schedule = model.OptimizeSchedule()
schedule = schedule.replace('\n', ' ')
expected_schedule = r'// Target: .+// MachineParams: .+// Delete this line if not using Generator Pipeline pipeline = get_pipeline\(\);.+Func OUT = pipeline.get_func\(1\);.+{.+}.+'
self.assertRegex(schedule, expected_schedule)
input = np.random.rand(2, 3) - 0.5
outputs = model.run([input])
self.assertEqual(1, len(outputs))
output = outputs[0]
expected = np.abs(input)
np.testing.assert_allclose(expected, output)
def test_onnx_to_caffe2_zipfile(self):
buf = tempfile.NamedTemporaryFile()
onnx_model = zipfile.ZipFile(buf, 'w')
output = tempfile.NamedTemporaryFile()
init_net_output = tempfile.NamedTemporaryFile()
node_def = helper.make_node(
"MatMul", ["X", "W"], ["Y"])
X = np.random.rand(2, 3).astype(np.float32)
W = np.random.rand(3, 2).flatten().astype(np.float32)
graph_def = helper.make_graph(
[node_def],
"test",
[helper.make_tensor_value_info("X", TensorProto.FLOAT, (2, 3)),
helper.make_tensor_value_info("W", TensorProto.FLOAT, (3, 2))],
[helper.make_tensor_value_info("Y", TensorProto.FLOAT, (2, 2))],
initializer=[helper.make_tensor("W",
TensorProto.FLOAT,
[3, 2],
b'__EXTERNAL',
raw=True)])
model_def = helper.make_model(graph_def, producer_name='onnx-to-caffe2-test')
onnx_model.writestr('__MODEL_PROTO', model_def.SerializeToString())
onnx_model.writestr('W', W.tobytes())
onnx_model.close()
W = W.reshape((3, 2))
Y_expect = np.matmul(X, W)
c2_model = c2.prepare_zip_archive(buf)
Y = c2_model.run(X).Y
np.testing.assert_allclose(Y, Y_expect)
def test_split_from_GLU(self): # type: () -> None
graph = self._make_graph(
[('x', TensorProto.FLOAT, (5, 6, 7))],
[make_node('Split', ['x'], ['y', 'z'], axis=1)],
[])
self._assert_inferred(graph, [make_tensor_value_info('y', TensorProto.FLOAT, (5, 3, 7)),
make_tensor_value_info('z', TensorProto.FLOAT, (5, 3, 7))])
def test_make_tensor_value_info(self): # type: () -> None
vi = helper.make_tensor_value_info('X', TensorProto.FLOAT, (2, 4))
checker.check_value_info(vi)
# scalar value
vi = helper.make_tensor_value_info('Y', TensorProto.FLOAT, ())
checker.check_value_info(vi)
def test_onnx_to_caffe2_loop(self):
body_nodes = [helper.make_node(
"MatMul", ["_X", "W"], ["_Y"])]
nodes = self._make_fake_loop_op(body_nodes,
[(TensorProto.FLOAT, (2, 2), "X")],
[(TensorProto.FLOAT, (2, 2), "Y")])
X = np.random.rand(2, 2).astype(np.float32)
W = np.random.rand(2, 2).flatten().astype(np.float32)
graph_def = helper.make_graph(
nodes,
"test",
[helper.make_tensor_value_info("X", TensorProto.FLOAT, (2, 2)),
helper.make_tensor_value_info("W", TensorProto.FLOAT, (2, 2))],
[helper.make_tensor_value_info("Y", TensorProto.FLOAT, (2, 2))],
initializer=[helper.make_tensor("W",
TensorProto.FLOAT,
[2, 2],
W.tolist())]
)
model_def = helper.make_model(graph_def, producer_name='onnx-to-caffe2-test')
Y = X
for _ in range(10):
Y = np.matmul(Y, W.reshape(2, 2))
p = c2.prepare(model_def)
out = p.run(X)
np.testing.assert_allclose(out.Y, Y)
def verify_constantfill(is_shape, input_dim, out_dim, value, dtype, **kwargs):
input_a = np.random.uniform(size=input_dim).astype(dtype)
out = np.empty(shape=out_dim, dtype=dtype)
out.fill(value)
if is_shape == True:
fill_node = helper.make_node("ConstantFill", [], ["out"], shape=input_dim, value=value, **kwargs)
else:
fill_node = helper.make_node("ConstantFill", ["input_a"], ["out"], value=value, dtype=dtype, **kwargs)
graph = helper.make_graph([fill_node],
"fill_test",
inputs = [helper.make_tensor_value_info("input_a",
TensorProto.FLOAT, list(input_dim))],
outputs = [helper.make_tensor_value_info("out",
TensorProto.FLOAT, list(out.shape))])
model = helper.make_model(graph, producer_name='fill_test')
for target, ctx in ctx_list():
if is_shape == True:
tvm_out = get_tvm_output(model, [], target, ctx, out.shape)
else:
tvm_out = get_tvm_output(model, [input_a], target, ctx, out.shape)
tvm.testing.assert_allclose(out, tvm_out, rtol=1e-5, atol=1e-5)
def verify_mean(input_dim):
dtype = 'float32'
a_np1 = np.random.uniform(size=input_dim).astype(dtype)
a_np2 = np.random.uniform(size=input_dim).astype(dtype)
a_np3 = np.random.uniform(size=input_dim).astype(dtype)
b_np = np.mean((a_np1, a_np2, a_np3), axis=0)
mean_node = helper.make_node("Mean", ["a_np1", "a_np2", "a_np3"], ["out"])
graph = helper.make_graph([mean_node],
"Mean_test",
inputs = [helper.make_tensor_value_info("a_np1",
TensorProto.FLOAT, list(input_dim)),
helper.make_tensor_value_info("a_np2",
TensorProto.FLOAT, list(input_dim)),
helper.make_tensor_value_info("a_np3",
TensorProto.FLOAT, list(input_dim))],
outputs = [helper.make_tensor_value_info("out",
TensorProto.FLOAT, list(b_np.shape))])
model = helper.make_model(graph, producer_name='Mean_test')
for target, ctx in ctx_list():
tvm_out = get_tvm_output(model, [a_np1, a_np2, a_np3], target, ctx, b_np.shape)
tvm.testing.assert_allclose(b_np, tvm_out, rtol=1e-5, atol=1e-5)
def verify_reduce_x(name, indata, axis, keepdims):
indata = np.array(indata).astype(np.float32)
# numpy expect result
if name == 'ReduceMax':
outdata = np.maximum.reduce(indata, axis=axis, keepdims=keepdims == 1)
elif name == 'ReduceMin':
outdata = np.minimum.reduce(indata, axis=axis, keepdims=keepdims == 1)
elif name == 'ReduceSum':
outdata = np.sum(indata, axis=axis, keepdims=keepdims == 1)
elif name == 'ReduceMean':
outdata = np.mean(indata, axis=axis, keepdims=keepdims == 1)
else:
raise Exception('unsupport op: {}'.format(name))
if len(np.asarray(outdata).shape) == 0:
outdata = np.asarray([outdata])
# onnx graph
if axis is None:
node = helper.make_node(name, inputs=['input'], outputs=['output'],
keepdims=keepdims)
else:
node = helper.make_node(name, inputs=['input'], outputs=['output'],
axis=axis, keepdims=keepdims)
graph = helper.make_graph([node],
'{}_test'.format(name),
inputs = [helper.make_tensor_value_info("input",
TensorProto.FLOAT, list(indata.shape))],
outputs = [helper.make_tensor_value_info("output",
TensorProto.FLOAT, list(outdata.shape))])
model = helper.make_model(graph, producer_name='{}_test'.format(name))
# tvm result
for target, ctx in ctx_list():
tvm_out = get_tvm_output(model, indata, target, ctx, outdata.shape, 'float32')
tvm.testing.assert_allclose(outdata, tvm_out, rtol=1e-5, atol=1e-5)
def verify_pad(indata, pads, value=0.0):
indata = np.array(indata).astype(np.float32)
# numpy expect result
len_dim = len(pads) // 2
np_pads = [(pads[i], pads[i+len_dim]) for i in range(len_dim)]
outdata = np.pad(indata, pad_width=np_pads, mode='constant', constant_values=value)
# onnx graph
node = helper.make_node(
'Pad',
inputs=['input'],
outputs=['output'],
mode='constant',
pads=pads,
value=value
)
graph = helper.make_graph([node],
'pad_test',
inputs = [helper.make_tensor_value_info("input",
TensorProto.FLOAT, list(indata.shape))],
outputs = [helper.make_tensor_value_info("output",
TensorProto.FLOAT, list(outdata.shape))])
model = helper.make_model(graph, producer_name='pad_test')
# tvm result
for target, ctx in ctx_list():
tvm_out = get_tvm_output(model, indata, target, ctx, outdata.shape, 'float32')
tvm.testing.assert_allclose(outdata, tvm_out, rtol=1e-5, atol=1e-5)
def test_shape():
in_shape = (4, 3, 3, 4)
ref_shape = (6, 2, 4, 3)
ref_array = np.array(ref_shape)
ref_node = onnx.helper.make_node('Constant',
inputs=[],
outputs=['ref_in'],
value=onnx.helper.make_tensor(name = 'const_tensor',
data_type = onnx.TensorProto.INT32,
dims = ref_array.shape,
vals = ref_array.flatten().astype(int)))
reshape_node = helper.make_node("Reshape", ["in", "ref_in"], ["out"])
shape_node = helper.make_node("Shape", ['out'], ['final_out'])
graph = helper.make_graph([ref_node, reshape_node, shape_node],
"shape_test",
inputs = [helper.make_tensor_value_info("in",
TensorProto.FLOAT, list(in_shape))],
outputs = [helper.make_tensor_value_info("final_out",
TensorProto.FLOAT, list(ref_shape))])
model = helper.make_model(graph, producer_name='shape_test')
for target, ctx in ctx_list():
x = np.random.uniform(size=in_shape).astype('int32')
tvm_out = get_tvm_output(model, x, target, ctx, ref_shape, 'int32')
tvm.testing.assert_allclose(ref_shape, tvm_out)
def test_eliminate_identity_single_use(self): # type: () -> None
nodes = [helper.make_node("Identity", ["X"], ["Y"])]
nodes.extend(self._make_fake_loop_op(
[helper.make_node("Identity", ["_Y"], ["_Y2"])],
[(TensorProto.FLOAT, (5,), "Y")],
[(TensorProto.FLOAT, (5,), "Y2")]))
graph = helper.make_graph(
nodes,
"test",
[helper.make_tensor_value_info("X", TensorProto.FLOAT, (5,))],
[helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5,)),
helper.make_tensor_value_info("Y2", TensorProto.FLOAT, (5,))])
optimized_model = self._optimized(graph, ["eliminate_identity"])
# All identity nodes should have been eliminated
def check_identity(node): # type: (NodeProto) -> None
assert node.op_type != "Identity"
self._visit_all_nodes_recursive(optimized_model.graph, check_identity)
# Use of the output from the Identity node in the main graph should
# have been replaced with the input to the identity node
assert len(optimized_model.graph.output) == 2
assert optimized_model.graph.output[0].name == "X"
# Use of the output from the Identity node in the loop graph should
# have been replaced with the input to that identity node
assert len(optimized_model.graph.node[2].attribute[0].g.output) == 2
assert optimized_model.graph.node[2].attribute[0].g.output[1].name == "_Y"
def _make_fake_loop_op(self,
body_nodes, # type: Sequence[NodeProto]
input_types, # type: Sequence[Tuple[TensorProto.DataType, Sequence[int], Text]]
output_types # type: Sequence[Tuple[TensorProto.DataType, Sequence[int], Text]]
): # type: (...) -> List[NodeProto]
zero = helper.make_tensor("trip_count_value", TensorProto.INT32, (), [10])
true = helper.make_tensor("condition", TensorProto.BOOL, (), [True])
# lcd is a dummy loop-carried dependency that only exists because
# right now the schema checker is broken and assumes a variadic
# input needs at least one value.
graph_inputs = [helper.make_tensor_value_info("i", TensorProto.INT32, ()),
helper.make_tensor_value_info("cond", TensorProto.BOOL, ())]
for type, shape, name in input_types:
graph_inputs.append(helper.make_tensor_value_info("_" + name, type, shape))
graph_outputs = [helper.make_tensor_value_info("cond", TensorProto.BOOL, ())]
for type, shape, name in output_types:
graph_outputs.append(helper.make_tensor_value_info("_" + name, type, shape))
body_graph = helper.make_graph(body_nodes, "body_graph", graph_inputs,
graph_outputs)
loop_inputs = ["trip_count", "condition"]
loop_inputs.extend([name for _, _, name in input_types])
# TODO: fix checker to accept 0-input variadic inputs
if len(loop_inputs) == 2:
loop_inputs.append("")
loop_outputs = [name for _, _, name in output_types]
retval_nodes = [
helper.make_node("Constant", [], ["trip_count"], value=zero),
helper.make_node("Constant", [], ["condition"], value=true),
helper.make_node("Loop", loop_inputs, loop_outputs, body=body_graph)
]
return retval_nodes
def test_reshape_like():
in_shape = (4, 3, 3, 4)
ref_shape = (3, 4, 4, 3)
ref_array = np.random.uniform(size=ref_shape).astype('float32')
ref_node = onnx.helper.make_node('Constant',
inputs=[],
outputs=['ref_in'],
value=onnx.helper.make_tensor(name = 'const_tensor',
data_type = onnx.TensorProto.FLOAT,
dims = ref_array.shape,
vals = ref_array.flatten().astype(float)))
copy_node = helper.make_node("Identity", ["ref_in"], ["copy_in"])
reshape_node = helper.make_node("Reshape", ["in", "copy_in"], ["out"])
graph = helper.make_graph([ref_node, copy_node, reshape_node],
"reshape_like_test",
inputs = [helper.make_tensor_value_info("in",
TensorProto.FLOAT, list(in_shape))],
outputs = [helper.make_tensor_value_info("out",
TensorProto.FLOAT, list(ref_shape))])
model = helper.make_model(graph, producer_name='reshape_like_test')
for target, ctx in ctx_list():
x = np.random.uniform(size=in_shape).astype('float32')
tvm_out = get_tvm_output(model, x, target, ctx, ref_shape, 'float32')
tvm.testing.assert_allclose(ref_shape, tvm_out.shape)
def test_fuse_transpose_into_gemm(self): # type: () -> None
nodes = [helper.make_node("Transpose", ["X"], ["A"], perm=[1, 0]),
helper.make_node("Transpose", ["Y"], ["B"], perm=[1, 0]),
helper.make_node("Gemm", ["A", "B", "C"], ["Z"])]
nodes.extend(self._make_fake_loop_op(
[helper.make_node("Transpose", ["_X"], ["_A"], perm=[1, 0]),
helper.make_node("Transpose", ["_Y"], ["_B"], perm=[1, 0]),
helper.make_node("Gemm", ["_A", "_B", "_C"], ["_Z2"])],
[(TensorProto.FLOAT, (2, 3), "X"),
(TensorProto.FLOAT, (5, 2), "Y"),
(TensorProto.FLOAT, (3, 5), "C")],
[(TensorProto.FLOAT, (2, 3), "Z2")]))
graph = helper.make_graph(
nodes,
"test",
[helper.make_tensor_value_info("X", TensorProto.FLOAT, (2, 3)),
helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5, 2)),
helper.make_tensor_value_info("C", TensorProto.FLOAT, (3, 5))],
[helper.make_tensor_value_info("Z", TensorProto.FLOAT, (3, 5))])
optimized_model = self._optimized(graph, ["fuse_transpose_into_gemm"])
# Gemm, Constant (trip count), Constant (cond), Loop
assert len(list(optimized_model.graph.node)) == 4
assert optimized_model.graph.node[0].op_type == "Gemm"
# Gemm
assert len(optimized_model.graph.node[3].attribute[0].g.node) == 1
assert optimized_model.graph.node[3].attribute[0].g.node[0].op_type == "Gemm"
def test_extract_constant_to_initializer(self): # type: () -> None
conv = helper.make_node("Conv", ["X", "Y"], ["Z"])
constant = helper.make_node("Constant", [], ["A"],
value=helper.make_tensor(
name="bias",
data_type=TensorProto.FLOAT,
dims=(16,),
vals=np.random.randn(16).astype(np.float32).tolist()))
add = helper.make_node("Add", ["Z", "A"], ["B"])
graph = helper.make_graph(
[conv, constant, add],
"test",
[helper.make_tensor_value_info("X", TensorProto.FLOAT, (1, 5, 3, 3)),
helper.make_tensor_value_info("Y", TensorProto.FLOAT, (16, 5, 3, 3))],
[helper.make_tensor_value_info("B", TensorProto.FLOAT, (1, 16, 3, 3))],
)
optimized_model = self._optimized(graph, ["extract_constant_to_initializer"])
self.assertEqual(
set(vi.name for vi in optimized_model.graph.input),
{'X', 'Y', 'A'})
self.assertEqual(len(optimized_model.graph.initializer), 1)
init = optimized_model.graph.initializer[0]
self.assertEqual(init.name, 'A')
self.assertEqual(init.dims, [16])
self.assertEqual(init.data_type, TensorProto.FLOAT)
self.assertEqual([n.op_type for n in optimized_model.graph.node], ['Conv', 'Add'])
def test_fuse_add_bias_into_conv_use_weight_shape(self): # type: () -> None
nodes = [helper.make_node("Conv", ["X", "Y"], ["Z"]),
helper.make_node("Add", ["Z", "A"], ["B"])]
nodes.extend(self._make_fake_loop_op(
[helper.make_node("Conv", ["_X", "_Y"], ["_Z"]),
helper.make_node("Add", ["_Z", "_A"], ["_B2"])],
[(TensorProto.FLOAT, (1, 5, 3, 3), "X"),
(TensorProto.FLOAT, (16, 5, 3, 3), "Y"),
(TensorProto.FLOAT, (16, 1, 1), "A")],
[(TensorProto.FLOAT, (1, 16, 3, 3), "B2")]))
graph = helper.make_graph(
nodes,
"test",
[helper.make_tensor_value_info("X", TensorProto.FLOAT, (1, 5, 3, 3)),
helper.make_tensor_value_info("Y", TensorProto.FLOAT, (16, 5, 3, 3)),
helper.make_tensor_value_info("A", TensorProto.FLOAT, (16, 1, 1))],
[helper.make_tensor_value_info("B", TensorProto.FLOAT, (1, 16, 3, 3))],
)
optimized_model = self._optimized(graph, ["fuse_add_bias_into_conv"])
# Squeeze, Conv, Constant (trip count), Constant (condition), Loop
assert len(list(optimized_model.graph.node)) == 5
assert optimized_model.graph.node[0].op_type == 'Squeeze'
assert optimized_model.graph.node[1].op_type == 'Conv'
assert optimized_model.graph.output[0].name == 'Z'
# Squeeze, Conv
assert len(optimized_model.graph.node[4].attribute[0].g.node) == 2
assert optimized_model.graph.node[4].attribute[0].g.node[0].op_type == 'Squeeze'
assert optimized_model.graph.node[4].attribute[0].g.node[1].op_type == 'Conv'
# Output 1 since 0 is 'cond'
assert optimized_model.graph.node[4].attribute[0].g.output[1].name == '_Z'
def test_fuse_add_bias_into_conv_use_move_constant(self): # type: () -> None
conv = helper.make_node("Conv", ["X", "Y"], ["Z"])
constant = helper.make_node("Constant", [], ["A"],
value=helper.make_tensor(
name="bias",
data_type=TensorProto.FLOAT,
dims=(16,),
vals=np.random.randn(16).astype(np.float32).tolist()))
add = helper.make_node("Add", ["Z", "A"], ["B"])
graph = helper.make_graph(
[conv, constant, add],
"test",
[helper.make_tensor_value_info("X", TensorProto.FLOAT, (1, 5, 3, 3)),
helper.make_tensor_value_info("Y", TensorProto.FLOAT, (16, 5, 3, 3))],
[helper.make_tensor_value_info("B", TensorProto.FLOAT, (1, 16, 3, 3))],
value_info=[
helper.make_tensor_value_info("A", TensorProto.FLOAT, (16, 1, 1)),
]
)
optimized_model = self._optimized(graph, ["fuse_add_bias_into_conv"])
assert len(optimized_model.graph.node) == 3
assert optimized_model.graph.node[0].op_type == 'Constant'
assert optimized_model.graph.node[1].op_type == 'Squeeze'
assert optimized_model.graph.node[2].op_type == 'Conv'
assert optimized_model.graph.output[0].name == 'Z'
assert optimized_model.graph.output[0].type.tensor_type.elem_type == TensorProto.FLOAT
assert len(optimized_model.graph.output[0].type.tensor_type.shape.dim) == 4
def test_lift_lex_if(self): # type: () -> None
nodes = [helper.make_node("Identity", ["X"], ["Y"])]
nodes.extend(self._make_fake_if_op(
[helper.make_node("Identity", ["X"], ["_Y2"]),
helper.make_node("Identity", ["Y"], ["_Y3"])],
[helper.make_node("Identity", ["X"], ["_Y2"]),
helper.make_node("Identity", ["X"], ["_Y3"])],
[(TensorProto.FLOAT, (5,), "Y2"),
(TensorProto.FLOAT, (5,), "Y3")]))
graph = helper.make_graph(
nodes,
"test",
[helper.make_tensor_value_info("X", TensorProto.FLOAT, (5,))],
[helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5,)),
helper.make_tensor_value_info("Y2", TensorProto.FLOAT, (5,))])
# "If" node now diverges from ONNX schema. Disable checking.
optimized_model = self._optimized(graph, ["lift_lexical_references"])
# Identity, Constant (condition), If
assert len(optimized_model.graph.node) == 3
# else_branch, then_branch, __control_inputs
assert len(optimized_model.graph.node[2].attribute) == 3
assert optimized_model.graph.node[2].attribute[2].name == "__control_inputs"
assert optimized_model.graph.node[2].attribute[2].strings[0] == b"X"
assert optimized_model.graph.node[2].attribute[2].strings[1] == b"Y"
def _test_upsample_bilinear_opset9():
scale = 2
in_shape = (1, 1, 3, 3)
out_shape = (1, 1, 3*scale, 3*scale)
y = helper.make_node("Upsample", ['in','scales'], ['out'], mode='linear')
scales=[1.0, 1.0, 2.0, 2.0]
in_array = np.random.uniform(size=in_shape).astype(np.float32)
out_array = topi.testing.bilinear_resize_python(in_array, (3*scale, 3*scale), "NCHW")
ref_array = np.array(scales)
ref_node = helper.make_node('Constant',
inputs=[],
outputs=['scales'],
value=onnx.helper.make_tensor(name = 'const_tensor',
data_type = TensorProto.FLOAT,
dims = ref_array.shape,
vals = ref_array.flatten().astype(float)))
graph = helper.make_graph([ref_node, y],
'upsample_bilinear_opset9_test',
inputs = [helper.make_tensor_value_info("in", TensorProto.FLOAT, list(in_shape))],
outputs = [helper.make_tensor_value_info("out", TensorProto.FLOAT, list(out_shape))])
model = helper.make_model(graph, producer_name='upsample_bilinear_opset9_test')
inputs = []
inputs.append(in_array)
for target, ctx in ctx_list():
tvm_out = get_tvm_output(model, inputs, target, ctx, out_shape, 'float32')
tvm.testing.assert_allclose(out_array, tvm_out, rtol=1e-5, atol=1e-5)
def test_eliminate_unused_initializer_no_eliminate_used(self): # type: () -> None
nodes = [helper.make_node("Add", ["X", "A"], ["Z"])]
nodes.extend(self._make_fake_loop_op(
[helper.make_node("Add", ["_X", "_A"], ["_Z2"])],
[(TensorProto.FLOAT, (1, 2), "X"),
(TensorProto.FLOAT, (1, 2), "A")],
[(TensorProto.FLOAT, (1, 2), "Z2")]))
graph = helper.make_graph(
nodes,
"test",
[helper.make_tensor_value_info("X", TensorProto.FLOAT, (1, 2)),
helper.make_tensor_value_info("A", TensorProto.FLOAT, (1, 2))],
[helper.make_tensor_value_info("Z", TensorProto.FLOAT, (1, 2))],
[helper.make_tensor("A", TensorProto.FLOAT,
dims=(1, 2),
vals=np.random.randn(1, 2).astype(np.float32).tobytes(),
raw=True)])
optimized_model = self._optimized(graph, ["eliminate_unused_initializer"])
# Add, Constant (trip count), Constant (cond), Loop
assert len(list(optimized_model.graph.node)) == 4
assert optimized_model.graph.node[0].op_type == "Add"
assert optimized_model.graph.output[0].name == "Z"
# Add
assert len(optimized_model.graph.node[3].attribute[0].g.node) == 1
assert optimized_model.graph.node[3].attribute[0].g.node[0].op_type == 'Add'
assert optimized_model.graph.node[3].attribute[0].g.output[1].name == '_Z2'
assert len(list(optimized_model.graph.initializer)) == 1
def _test_power_iteration(x_shape, y_shape):
if isinstance(y_shape, int):
y_shape = [y_shape]
x = np.random.uniform(size=x_shape).astype(np.float32)
y = np.random.uniform(size=y_shape).astype(np.float32)
np_res = np.power(x, y).astype(np.float32)
res = helper.make_node("Pow", ['x', 'y'], ['out'])
graph = helper.make_graph([res],
'power_test',
inputs = [helper.make_tensor_value_info("x",
TensorProto.FLOAT, list(x_shape)),
helper.make_tensor_value_info("y",
TensorProto.FLOAT, list(y_shape))],
outputs = [helper.make_tensor_value_info("out",
TensorProto.FLOAT, list(np_res.shape))])
model = helper.make_model(graph, producer_name='power_test')
for target, ctx in ctx_list():
tvm_out = get_tvm_output(model, [x, y], target, ctx, np_res.shape)
tvm.testing.assert_allclose(np_res, tvm_out, rtol=1e-5, atol=1e-5)
def test_initializer(self):
X = np.array([[1, 2], [3, 4]]).astype(np.float32)
Y = np.array([[1, 2], [3, 4]]).astype(np.float32)
weight = np.array([[1, 0], [0, 1]])
graph_def = make_graph(
[make_node("Add", ["X", "Y"], ["Z0"]),
make_node("Cast", ["Z0"], ["Z"], to="float"),
make_node("Mul", ["Z", "weight"], ["W0"]),
make_node("Tanh", ["W0"], ["W1"]),
make_node("Sigmoid", ["W1"], ["W2"]),
make_node("Scale", ["W2"], ["W3"], scale=-1.0)],
name="test_initializer",
inputs=[
make_tensor_value_info("X", onnx.TensorProto.FLOAT, (2, 2)),
make_tensor_value_info("Y", onnx.TensorProto.FLOAT, (2, 2)),
make_tensor_value_info("weight", onnx.TensorProto.FLOAT, (2, 2)),
],
outputs=[
make_tensor_value_info("W3", onnx.TensorProto.FLOAT, (2, 2))
],
initializer=[make_tensor("weight",
onnx.TensorProto.FLOAT,
[2, 2],
weight.flatten().astype(float))]
)
def sigmoid(x):
return 1 / (1 + np.exp(-x))
W_ref = -sigmoid(np.tanh((X + Y) * weight))
c2_rep = c2.prepare(make_model(graph_def, producer_name='caffe2-ref-test'))
output = c2_rep.run({"X": X, "Y": Y})
np.testing.assert_almost_equal(output["W3"], W_ref)
def verify_lrn(shape, nsize, dtype, alpha=None, beta=None, bias=None):
in_array = np.random.uniform(size=shape).astype(dtype)
if alpha == None and beta == None and bias==None:
alpha = 0.0001
beta = 0.75
bias = 1.0
node = onnx.helper.make_node('LRN', inputs=['in'], outputs=['out'], size=nsize)
else:
node = onnx.helper.make_node('LRN', inputs=['in'], outputs=['out'], alpha=alpha,
beta=beta, bias=bias, size=nsize)
graph = helper.make_graph([node],
"lrn_test",
inputs = [helper.make_tensor_value_info("in", TensorProto.FLOAT, list(shape))],
outputs = [helper.make_tensor_value_info("out", TensorProto.FLOAT, list(shape))])
model = helper.make_model(graph, producer_name='lrn_test')
def _get_python_lrn():
square_sum = np.zeros(shape).astype(dtype)
for n, c, h, w in np.ndindex(in_array.shape):
square_sum[n, c, h, w] = sum(in_array[n,
max(0, c - int(math.floor((nsize - 1) / 2))): \
min(5, c + int(math.ceil((nsize - 1) / 2)) + 1),
h,
w] ** 2)
py_out = in_array / ((bias + (alpha / nsize) * square_sum) ** beta)
return py_out
for target, ctx in ctx_list():
input_name = model.graph.input[0].name
py_out = _get_python_lrn()
tvm_out = get_tvm_output(model, in_array, target, ctx, py_out.shape, 'float32')
tvm.testing.assert_allclose(py_out, tvm_out, rtol=1e-5, atol=1e-5)
def verify_split(indata, outdatas, split, axis=0):
indata = np.array(indata).astype(np.float32)
outdatas = [np.array(o).astype(np.float32) for o in outdatas]
node = helper.make_node(
'Split',
inputs=['input'],
outputs=['output_{}'.format(i) for i in range(len(split))],
axis=axis,
split=split
)
graph = helper.make_graph([node],
'split_test',
inputs = [helper.make_tensor_value_info("input",
TensorProto.FLOAT, list(indata.shape))],
outputs = [helper.make_tensor_value_info("output_{}".format(i),
TensorProto.FLOAT, list(outdatas[i].shape))
for i in range(len(split))
])
model = helper.make_model(graph, producer_name='split_test')
for target, ctx in ctx_list():
output_shape = [o.shape for o in outdatas]
output_type = ['float32', 'float32', 'float32']
tvm_out = get_tvm_output(model, indata, target, ctx, output_shape, output_type)
for o, t in zip(outdatas, tvm_out):
tvm.testing.assert_allclose(o, t)
def test_scalars(self):
# Create 2 inputs
X = helper.make_tensor_value_info('A', TensorProto.INT32, [])
Y = helper.make_tensor_value_info('B', TensorProto.INT32, [])
# Create one output
Z = helper.make_tensor_value_info('C', TensorProto.INT32, [])
# Create a node
node_def = helper.make_node('Add', ['A', 'B'], ['C'])
# Create the model
graph_def = helper.make_graph([node_def], "scalar-model", [X, Y], [Z])
onnx_model = helper.make_model(graph_def,
producer_name='onnx-example')
model = Model()
model.BuildFromOnnxModel(onnx_model)
schedule = model.OptimizeSchedule()
schedule = schedule.replace('\n', ' ')
expected_schedule = r'// Target: .+// MachineParams: .+// Delete this line if not using Generator Pipeline pipeline = get_pipeline\(\);.+Func C = pipeline.get_func\(2\);.+{.+}.+'
self.assertRegex(schedule, expected_schedule)
input1 = np.random.randint(-10, 10, size=())
input2 = np.random.randint(-10, 10, size=())
outputs = model.run([input1, input2])
self.assertEqual(1, len(outputs))
output = outputs[0]
expected = input1 + input2
np.testing.assert_allclose(expected, output)
def test_topk(self): # type: () -> None
graph = self._make_graph(
[('x', TensorProto.FLOAT, (3, 4, 5, 10))],
[make_node('TopK', ['x'], ['y', 'z'], k=2, axis=2)],
[])
self._assert_inferred(graph,
[make_tensor_value_info('y', TensorProto.FLOAT, (3, 4, 2, 10)),
make_tensor_value_info('z', TensorProto.INT64, (3, 4, 2, 10))])
def test_GLU_partial(self): # type: () -> None
graph = self._make_graph(
[('x', TensorProto.FLOAT, (5, 6, 7))],
[make_node('Split', ['x'], ['y', 'z'], axis=1),
make_node('Sigmoid', ['z'], ['a'])],
[])
self._assert_inferred(graph, [make_tensor_value_info('y', TensorProto.FLOAT, (5, 3, 7)),
make_tensor_value_info('z', TensorProto.FLOAT, (5, 3, 7)),
make_tensor_value_info('a', TensorProto.FLOAT, (5, 3, 7))])
def _rnn_forward(self, seqlen, batchsize, inpsize, hiddensize): # type: (int, int, int, int) -> None
graph = self._make_graph(
[('x', TensorProto.FLOAT, (seqlen, batchsize, inpsize)),
('w', TensorProto.FLOAT, (1, hiddensize, inpsize)),
('r', TensorProto.FLOAT, (1, hiddensize, hiddensize))],
[make_node('RNN', ['x', 'w', 'r'], ['all', 'last'], hidden_size=hiddensize)],
[])
self._assert_inferred(graph, [
make_tensor_value_info('all', TensorProto.FLOAT, (seqlen, 1, batchsize, hiddensize)),
make_tensor_value_info('last', TensorProto.FLOAT, (1, batchsize, hiddensize))])
def test_matmul_integer(self):
if legacy_opset_pre_ver(10):
raise unittest.SkipTest(
"ONNX version {} doesn't support MatMulInteger.".format(
defs.onnx_opset_version()))
node_def = helper.make_node("MatMulInteger",
["A", "B", "a_zero_point", "b_zero_point"],
["Z"])
# A & B are 3-D tensor and a_zero_point & b_zero_point are scalar
A = self._get_rnd_int(-20, 20, shape=(2, 3, 4), dtype=np.int8)
B = self._get_rnd_int(-20, 20, shape=(2, 4, 6), dtype=np.int8)
a_zero_point = self._get_rnd_int(-20, 20, dtype=np.int8)
b_zero_point = self._get_rnd_int(-20, 20, dtype=np.int8)
A_minus_zero_point = np.subtract(A.astype(np.int32),
a_zero_point.astype(np.int32))
B_minus_zero_point = np.subtract(B.astype(np.int32),
b_zero_point.astype(np.int32))
z = np.matmul(A_minus_zero_point, B_minus_zero_point)
graph_def = helper.make_graph(
[node_def],
name="test_unknown_shape",
inputs=[
helper.make_tensor_value_info("A", TensorProto.INT8,
[None, None, None]),
helper.make_tensor_value_info("B", TensorProto.INT8,
[None, None, None]),
helper.make_tensor_value_info("a_zero_point", TensorProto.INT8,
[]),
helper.make_tensor_value_info("b_zero_point", TensorProto.INT8,
[])
],
outputs=[
helper.make_tensor_value_info("Z", TensorProto.INT32,
[None, None, None])
])
tf_rep = onnx_graph_to_tensorflow_rep(graph_def)
output = tf_rep.run({
"A": A,
"B": B,
"a_zero_point": a_zero_point,
"b_zero_point": b_zero_point
})
np.testing.assert_almost_equal(output["Z"], z)
# A & B are 4-D tensor and a_zero_point & b_zero_point are 1-D tensor
A = self._get_rnd_int(-20, 20, shape=(2, 5, 3, 4), dtype=np.int8)
B = self._get_rnd_int(-20, 20, shape=(2, 1, 4, 6), dtype=np.int8)
a_zero_point = self._get_rnd_int(-20,
20,
shape=(A.shape[-2]),
dtype=np.int8)
b_zero_point = self._get_rnd_int(-20,
20,
shape=(B.shape[-1]),
dtype=np.int8)
a_zero_point_with_reshape = np.reshape(a_zero_point, [A.shape[-2], 1])
A_minus_zero_point = np.subtract(
A.astype(np.int32), a_zero_point_with_reshape.astype(np.int32))
B_minus_zero_point = np.subtract(B.astype(np.int32),
b_zero_point.astype(np.int32))
z = np.matmul(A_minus_zero_point, B_minus_zero_point)
graph_def = helper.make_graph(
[node_def],
name="test_unknown_shape",
inputs=[
helper.make_tensor_value_info("A", TensorProto.INT8,
[None, None, None, None]),
helper.make_tensor_value_info("B", TensorProto.INT8,
[None, None, None, None]),
helper.make_tensor_value_info("a_zero_point", TensorProto.INT8,
[None]),
helper.make_tensor_value_info("b_zero_point", TensorProto.INT8,
[None])
],
outputs=[
helper.make_tensor_value_info("Z", TensorProto.INT32,
[None, None, None, None])
])
tf_rep = onnx_graph_to_tensorflow_rep(graph_def)
output = tf_rep.run({
"A": A,
"B": B,
"a_zero_point": a_zero_point,
"b_zero_point": b_zero_point
})
np.testing.assert_almost_equal(output["Z"], z)
def parseModel(self, domain, graph, expInput=None, expOutput=None):
order, expanded_order = graph.topologicalSort()
index = 0
#Inputs
inputs = list()
#inits
initializers = list()
#Nodes
nodes = list()
# print(len(expanded_order))
for id, value in expanded_order.items():
index = index + 1
node = graph.nodes.get(id)
name = node.type
m = node.params
l = node.learned_params
# print(id, index, name, m)
# it's important that the values in input and output are all string
# layerName = prefix+"_"+str(id)
layerName = str(id)
input = list(map(str, value['prior']))
if expOutput:
expOutputNum = str(expOutput[0])
input = [expOutputNum + "_" + s for s in input]
layerName = expOutputNum + "_" + layerName
if expInput:
if len(input) == 0:
input = expInput
if expOutput and index == len(expanded_order):
output = expOutput
#XXX This handles the output of nested sequential modules
# to come back into normal flow of ids
layerName = expOutputNum #XXX
else:
output = [layerName]
# for first node add a dummy input
if len(input) == 0:
input = ['X']
inputs.append(
helper.make_tensor_value_info('X', TensorProto.FLOAT, [1]))
if m.get('in_channels'):
in_channels = m.get('in_channels')
else:
in_channels = 1
if m.get('out_channels'):
out_channels = m.get('out_channels')
else:
out_channels = 1
if m.get('num_features'):
num_features = m.get('num_features')
else:
num_features = 1
if m.get('in_features'):
in_features = m.get('in_features')
else:
in_features = 1
if m.get('out_features'):
out_features = m.get('out_features')
else:
out_features = 1
if m.get('p'):
ratio = m.get('p')
else:
ratio = 0.1
if m.get('kernel_size'):
kernel_size = m.get('kernel_size')
if type(kernel_size) == tuple:
kernel_size = list(kernel_size)
else:
kernel_size = [kernel_size, kernel_size]
else:
kernel_size = [1, 1]
if m.get('stride'):
stride = m.get('stride')
if type(stride) == tuple:
stride = list(stride)
else:
stride = [stride, stride]
else:
stride = [1, 1]
if m.get('padding'):
padding = m.get('padding')
if type(padding) == tuple:
padding = list(padding)
padding.extend(padding)
else:
padding = [padding, padding, padding, padding]
else:
padding = [0, 0, 0, 0]
if m.get('subgraph'):
subgraph = m.get('subgraph')
# copy the network weights
weight = l.get('weight')
bias = l.get('bias')
running_mean = l.get('running_mean')
running_var = l.get('running_var')
#helper.make_tensor(name='a',data_type=TensorProto.FLOAT, dims=(1,), vals=np.ones(1).tolist())
if name == 'Conv2d':
learned_input = [
"learned_" + layerName + "_W",
"learned_" + layerName + "_B"
]
input.extend(learned_input)
# declare learned parameters
inputs.append(
helper.make_tensor_value_info(
"learned_" + layerName + "_W", TensorProto.FLOAT, [
out_channels, in_channels, kernel_size[0],
kernel_size[1]
]))
inputs.append(
helper.make_tensor_value_info(
"learned_" + layerName + "_B", TensorProto.FLOAT,
[out_channels]))
# initialize learned parameters with existing weights
if self.isValid(weight):
initializers.append(
helper.make_tensor("learned_" + layerName + "_W",
TensorProto.FLOAT,
dims=[
out_channels, in_channels,
kernel_size[0], kernel_size[1]
],
vals=self.toTensor(weight)))
if self.isValid(bias):
initializers.append(
helper.make_tensor("learned_" + layerName + "_B",
TensorProto.FLOAT,
dims=[out_channels],
vals=self.toTensor(bias)))
nodes.append(
helper.make_node("Conv",
input,
output,
name=layerName,
domain=domain,
kernel_shape=kernel_size,
strides=stride,
pads=padding,
dilations=[1, 1],
group=1))
elif name == 'BatchNorm2d':
learned_input = [
"learned_" + layerName + "_W",
"learned_" + layerName + "_B",
"learned_" + layerName + "_M",
"learned_" + layerName + "_V"
]
input.extend(learned_input)
# declare learned parameters
inputs.append(
helper.make_tensor_value_info(
"learned_" + layerName + "_W", TensorProto.FLOAT,
[num_features]))
inputs.append(
helper.make_tensor_value_info(
"learned_" + layerName + "_B", TensorProto.FLOAT,
[num_features]))
inputs.append(
helper.make_tensor_value_info(
"learned_" + layerName + "_M", TensorProto.FLOAT,
[num_features]))
inputs.append(
helper.make_tensor_value_info(
"learned_" + layerName + "_V", TensorProto.FLOAT,
[num_features]))
# initialize learned parameters with existing weights
if self.isValid(weight):
initializers.append(
helper.make_tensor("learned_" + layerName + "_W",
TensorProto.FLOAT, [num_features],
vals=self.toTensor(weight)))
if self.isValid(bias):
initializers.append(
helper.make_tensor("learned_" + layerName + "_B",
TensorProto.FLOAT, [num_features],
vals=self.toTensor(bias)))
if self.isValid(running_mean):
initializers.append(
helper.make_tensor("learned_" + layerName + "_M",
TensorProto.FLOAT, [num_features],
vals=self.toTensor(running_mean)))
if self.isValid(running_var):
initializers.append(
helper.make_tensor("learned_" + layerName + "_V",
TensorProto.FLOAT, [num_features],
vals=self.toTensor(running_var)))
nodes.append(
helper.make_node("BatchNormalization",
input,
output,
name=layerName,
domain=domain,
epsilon=9.99999974737875e-06,
momentum=1))
elif name == 'ReLU':
nodes.append(
helper.make_node("Relu",
input,
output,
name=layerName,
domain=domain))
elif name == 'Sigmoid':
nodes.append(
helper.make_node("Sigmoid",
input,
output,
name=layerName,
domain=domain))
elif name == 'MaxPool2d':
nodes.append(
helper.make_node("MaxPool",
input,
output,
name=layerName,
domain=domain,
kernel_shape=[3, 3],
pads=[0, 0, 0, 0],
strides=[2, 2]))
elif name == 'AvgPool2d':
nodes.append(
helper.make_node("AveragePool",
input,
output,
name=layerName,
domain=domain,
kernel_shape=[3, 3],
pads=[0, 0, 0, 0],
strides=[2, 2]))
elif name == 'Linear':
learned_input = [layerName + "_W", layerName + "_B"]
input.extend(learned_input)
# declare learned parameters
inputs.append(
helper.make_tensor_value_info(
"learned_" + layerName + "_W", TensorProto.FLOAT,
[out_features, in_features]))
inputs.append(
helper.make_tensor_value_info(
"learned_" + layerName + "_B", TensorProto.FLOAT,
[out_features]))
# initialize learned parameters with existing weights
if self.isValid(weight):
initializers.append(
helper.make_tensor("learned_" + layerName + "_W",
TensorProto.FLOAT,
[out_features, in_features],
vals=self.toTensor(weight)))
if self.isValid(bias):
initializers.append(
helper.make_tensor("learned_" + layerName + "_B",
TensorProto.FLOAT, [out_features],
vals=self.toTensor(bias)))
nodes.append(
helper.make_node("Gemm",
input,
output,
name=layerName,
domain=domain,
alpha=1,
beta=1,
transB=1))
elif name == 'Dropout':
nodes.append(
helper.make_node("Dropout",
input,
output,
name=layerName,
domain=domain,
ratio=ratio))
elif name == 'Softmax':
nodes.append(
helper.make_node("Softmax",
input,
output,
name=layerName,
domain=domain))
elif name == 'Identity':
nodes.append(
helper.make_node("Identity",
input,
output,
name=layerName,
domain=domain))
elif name == 'Reshape':
nodes.append(
helper.make_node("Reshape",
input,
output,
name=layerName,
domain=domain))
elif name == 'BCELoss':
nodes.append(
helper.make_node(name,
input,
output,
name=layerName,
domain=domain))
elif name == 'MSELoss':
nodes.append(
helper.make_node(name,
input,
output,
name=layerName,
domain=domain))
elif name == 'Sequential':
sub_nodes, sub_inputs, sub_outputs, sub_initializers = self.parseModel(
domain, subgraph, input, output)
# onnx_graph = helper.make_graph(sub_nodes, model_name", sub_inputs, sub_outputs)
# print(helper.printable_graph(onnx_graph))
nodes.extend(sub_nodes)
inputs.extend(sub_inputs)
initializers.extend(sub_initializers)
else:
# Group or Sequential nodes
if node.group == True:
sub_nodes, sub_inputs, sub_outputs, sub_initializers = self.parseModel(
domain, subgraph, input, output)
nodes.extend(sub_nodes)
inputs.extend(sub_inputs)
initializers.extend(sub_initializers)
else:
print('Not Implement', name)
# Outputs
outputs = list()
outputs.append(
helper.make_tensor_value_info(layerName, TensorProto.FLOAT, [1]))
return nodes, inputs, outputs, initializers
def test_sequence_ops(self):
# test SequenceConstruct and SequenceAt
a = np.random.randn(2, 1, 2).astype(np.float32)
b = np.random.randn(1, 1, 2).astype(np.float32)
c = np.random.randn(3, 1, 2).astype(np.float32)
seq_construct_node = helper.make_node('SequenceConstruct', ['a', 'b', 'c'],
['S'])
seq_at_node = helper.make_node('SequenceAt', ['S', 'at'], ['Y'])
out_value_info = helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT,
[None])
a_value_info = helper.make_tensor_value_info('a', onnx.TensorProto.FLOAT,
[2, 1, 2])
b_value_info = helper.make_tensor_value_info('b', onnx.TensorProto.FLOAT,
[1, 1, 2])
c_value_info = helper.make_tensor_value_info('c', onnx.TensorProto.FLOAT,
[3, 1, 2])
at_value_info = helper.make_tensor_value_info('at', onnx.TensorProto.INT32,
[])
graph = helper.make_graph(
[seq_construct_node, seq_at_node],
name='seq_construct_at_test',
inputs=[a_value_info, b_value_info, c_value_info, at_value_info],
outputs=[out_value_info])
model = helper.make_model(graph, producer_name='backend-test')
tf_rep = prepare(model)
output = tf_rep.run({'a': a, 'b': b, 'c': c, 'at': 0})
np.testing.assert_almost_equal(output["Y"], a)
output = tf_rep.run({'a': a, 'b': b, 'c': c, 'at': -2})
np.testing.assert_almost_equal(output["Y"], b)
output = tf_rep.run({'a': a, 'b': b, 'c': c, 'at': 2})
np.testing.assert_almost_equal(output["Y"], c)
# test SequenceEmpty, SequenceInsert, and SequenceAt
p = np.int32(0)
seq_empty_node = helper.make_node('SequenceEmpty', [], ['S'])
seq_insert_node1 = helper.make_node('SequenceInsert', ['S', 'a'], ['S1'])
seq_insert_node2 = helper.make_node('SequenceInsert', ['S1', 'b'], ['S2'])
seq_insert_node3 = helper.make_node('SequenceInsert', ['S2', 'c', 'p'],
['S3'])
seq_at_node = helper.make_node('SequenceAt', ['S3', 'at'], ['Y'])
p_value_info = helper.make_tensor_value_info('p', onnx.TensorProto.INT32,
[])
graph = helper.make_graph([
seq_empty_node, seq_insert_node1, seq_insert_node2, seq_insert_node3,
seq_at_node
],
name='seq_empty_insert_at_test',
inputs=[
a_value_info, b_value_info, c_value_info,
p_value_info, at_value_info
],
outputs=[out_value_info])
model = helper.make_model(graph, producer_name='backend-test')
tf_rep = prepare(model)
output = tf_rep.run({'a': a, 'b': b, 'c': c, 'p': p, 'at': 0})
np.testing.assert_almost_equal(output["Y"], c)
# test SequenceConstruct, SequenceErase, and SequenceLength
seq_construct_node = helper.make_node('SequenceConstruct', ['a', 'b', 'c'],
['S'])
seq_erase_node = helper.make_node('SequenceErase', ['S', 'p'], ['S1'])
seq_length_node = helper.make_node('SequenceLength', ['S1'], ['Y'])
graph = helper.make_graph(
[seq_construct_node, seq_erase_node, seq_length_node],
name='seq_construct_erase_length_test',
inputs=[a_value_info, b_value_info, c_value_info, p_value_info],
outputs=[out_value_info])
model = helper.make_model(graph, producer_name='backend-test')
tf_rep = prepare(model)
output = tf_rep.run({'a': a, 'b': b, 'c': c, 'p': p})
np.testing.assert_almost_equal(output["Y"], 2)
# test SequenceConstruct and SequenceErase
seq_construct_node = helper.make_node('SequenceConstruct', ['a', 'b', 'c'],
['S'])
seq_erase_node = helper.make_node('SequenceErase', ['S', 'p'], ['S1'])
seq_at_node = helper.make_node('SequenceAt', ['S1', 'at'], ['Y'])
graph = helper.make_graph([seq_construct_node, seq_erase_node, seq_at_node],
name='seq_construct_erase_test',
inputs=[
a_value_info, b_value_info, c_value_info,
p_value_info, at_value_info
],
outputs=[out_value_info])
model = helper.make_model(graph, producer_name='backend-test')
tf_rep = prepare(model)
output = tf_rep.run({'a': a, 'b': b, 'c': c, 'p': p, 'at': 0})
np.testing.assert_almost_equal(output["Y"], b)
output = tf_rep.run({'a': a, 'b': b, 'c': c, 'p': p, 'at': 1})
np.testing.assert_almost_equal(output["Y"], c)
# test SequenceConstruct and ConcatFromSequence
seq_construct_node = helper.make_node('SequenceConstruct', ['a', 'b', 'c'],
['S'])
concat_from_seq_node = helper.make_node('ConcatFromSequence', ['S'], ['Y'],
axis=1)
a = np.array([[1, 2],[3, 4]]).astype(np.float32)
b = np.array([[5, 6],[7, 8]]).astype(np.float32)
c = np.array([[9, 10],[11, 12]]).astype(np.float32)
a_value_info = helper.make_tensor_value_info('a', onnx.TensorProto.FLOAT,
[2, 2])
b_value_info = helper.make_tensor_value_info('b', onnx.TensorProto.FLOAT,
[2, 2])
c_value_info = helper.make_tensor_value_info('c', onnx.TensorProto.FLOAT,
[2, 2])
graph = helper.make_graph([seq_construct_node, concat_from_seq_node],
name='seq_construct_concat_test',
inputs=[a_value_info, b_value_info, c_value_info],
outputs=[out_value_info])
model = helper.make_model(graph, producer_name='backend-test')
tf_rep = prepare(model)
output = tf_rep.run({'a': a, 'b': b, 'c': c})
d = np.concatenate((a, b, c), axis=1).astype(np.float32)
np.testing.assert_almost_equal(output["Y"], d)
# test SplitToSequence and SequenceAt
a = np.array([[1, 2, 3, 4, 5, 6, 7], [11, 12, 13, 14, 15, 16, 17],
[21, 22, 23, 24, 25, 26, 27]]).astype(np.float32)
b = np.int32([2, 1])
seq_split_node = helper.make_node('SplitToSequence', ['a', 'b'], ['S'])
seq_at_node = helper.make_node('SequenceAt', ['S', 'at'], ['Y'])
a_value_info = helper.make_tensor_value_info('a', onnx.TensorProto.FLOAT,
[3, 7])
b_value_info = helper.make_tensor_value_info('b', onnx.TensorProto.INT32,
[2])
at_value_info = helper.make_tensor_value_info('at', onnx.TensorProto.INT32,
[])
graph = helper.make_graph(
[seq_split_node, seq_at_node],
name='split_to_seq_test',
inputs=[a_value_info, b_value_info, at_value_info],
outputs=[out_value_info])
model = helper.make_model(graph, producer_name='backend-test')
tf_rep = prepare(model)
output = tf_rep.run({'a': a, 'b': b, 'at': 1})
np.testing.assert_almost_equal(output["Y"], np.split(a, [2, 3])[1])
axis = 1
seq_split_node = helper.make_node('SplitToSequence', ['a'], ['S'],
axis=axis)
seq_at_node = helper.make_node('SequenceAt', ['S', 'at'], ['Y'])
at_value_info = helper.make_tensor_value_info('at', onnx.TensorProto.INT32,
[])
graph = helper.make_graph([seq_split_node, seq_at_node],
name='split_to_seq_test',
inputs=[a_value_info, at_value_info],
outputs=[out_value_info])
model = helper.make_model(graph, producer_name='backend-test')
tf_rep = prepare(model)
output = tf_rep.run({'a': a, 'at': 0})
np.testing.assert_almost_equal(output["Y"], np.split(a, 7, axis=1)[0])
seq_split_node = helper.make_node('SplitToSequence', ['a'], ['S'],
keepdims=0)
seq_at_node = helper.make_node('SequenceAt', ['S', 'at'], ['Y'])
at_value_info = helper.make_tensor_value_info('at', onnx.TensorProto.INT32,
[])
graph = helper.make_graph([seq_split_node, seq_at_node],
name='split_to_seq_test',
inputs=[a_value_info, at_value_info],
outputs=[out_value_info])
model = helper.make_model(graph, producer_name='backend-test')
tf_rep = prepare(model)
output = tf_rep.run({'a': a, 'at': 0})
expected = [np.squeeze(res) for res in np.split(a, 3)]
np.testing.assert_almost_equal(output["Y"], expected[0])
def create_reduce_lp(self, shape, axes, keep_dims, reduce_p, ir_version):
"""
ONNX net IR net
Input->ReduceLX(axes)->Output => Input->ReduceLX
"""
#
# Create ONNX model
#
import onnx
from onnx import helper
from onnx import TensorProto
output_shape = shape.copy()
_axes = axes.copy() if axes is not None else list(range(len(shape)))
for axis in _axes:
output_shape[axis] = 1
if not keep_dims:
output_shape = [dim for dim in output_shape if dim != 1]
input = helper.make_tensor_value_info('input', TensorProto.FLOAT, shape)
output = helper.make_tensor_value_info('output', TensorProto.FLOAT, output_shape)
args = dict(keepdims=keep_dims)
if axes:
args['axes'] = axes
node_def = onnx.helper.make_node(
"ReduceL" + str(reduce_p),
inputs=['input'],
outputs=['output'],
**args
)
# Create the graph (GraphProto)
graph_def = helper.make_graph(
[node_def],
'test_model',
[input],
[output],
)
# Create the model (ModelProto)
onnx_net = helper.make_model(graph_def, producer_name='test_model')
#
# Create reference IR net
# Please, specify 'type': 'Input' for input node
# Moreover, do not forget to validate ALL layer attributes!!!
#
ref_net = None
if check_ir_version(10, None, ir_version):
nodes_attributes = {
'input': {'kind': 'op', 'type': 'Parameter'},
'input_data': {'shape': shape, 'kind': 'data'},
'input_data_1': {'shape': [len(_axes)], 'value': _axes, 'kind': 'data'},
'const_1': {'kind': 'op', 'type': 'Const'},
'const_data_1': {'shape': [len(_axes)], 'kind': 'data'},
'reduce': {'kind': 'op', 'type': "ReduceL" + str(reduce_p), 'keep_dims': keep_dims},
'reduce_data': {'shape': output_shape, 'kind': 'data'},
'result': {'kind': 'op', 'type': 'Result'}
}
ref_net = build_graph(nodes_attributes,
[('input', 'input_data'),
('input_data_1', 'const_1'),
('const_1', 'const_data_1'),
('input_data', 'reduce'),
('const_data_1', 'reduce'),
('reduce', 'reduce_data'),
('reduce_data', 'result')
])
return onnx_net, ref_net
def test_cast_errors():
from onnx.onnx_cpp2py_export.checker import ValidationError
np.random.seed(133391)
input_data = np.ceil(np.random.rand(2, 3, 4) * 16)
# missing 'to' attribute
node = onnx.helper.make_node("Cast", inputs=["A"], outputs=["B"])
input_tensors = [
make_tensor_value_info(name, onnx.TensorProto.FLOAT, value.shape)
for name, value in zip(node.input, [input_data])
]
output_tensors = [
make_tensor_value_info(node.output[0], onnx.TensorProto.FLOAT16,
input_data.shape)
] # type: ignore
graph = make_graph([node], "compute_graph", input_tensors, output_tensors)
model = make_model(graph, producer_name="NgraphBackend")
with pytest.raises(ValidationError):
import_onnx_model(model)
# unsupported data type representation
node = onnx.helper.make_node("Cast",
inputs=["A"],
outputs=["B"],
to=1.2345)
input_tensors = [
make_tensor_value_info(name, onnx.TensorProto.FLOAT, value.shape)
for name, value in zip(node.input, [input_data])
]
output_tensors = [
make_tensor_value_info(node.output[0], onnx.TensorProto.INT32,
input_data.shape)
] # type: ignore
graph = make_graph([node], "compute_graph", input_tensors, output_tensors)
model = make_model(graph, producer_name="NgraphBackend")
with pytest.raises(ValidationError):
import_onnx_model(model)
# unsupported input tensor data type:
node = onnx.helper.make_node("Cast",
inputs=["A"],
outputs=["B"],
to=onnx.TensorProto.INT32)
input_tensors = [
make_tensor_value_info(name, onnx.TensorProto.COMPLEX64, value.shape)
for name, value in zip(node.input, [input_data])
]
output_tensors = [
make_tensor_value_info(node.output[0], onnx.TensorProto.INT32,
input_data.shape)
] # type: ignore
graph = make_graph([node], "compute_graph", input_tensors, output_tensors)
model = make_model(graph, producer_name="NgraphBackend")
with pytest.raises((RuntimeError, OVTypeError)):
import_onnx_model(model)
# unsupported output tensor data type:
node = onnx.helper.make_node("Cast",
inputs=["A"],
outputs=["B"],
to=onnx.TensorProto.COMPLEX128)
input_tensors = [
make_tensor_value_info(name, onnx.TensorProto.FLOAT, value.shape)
for name, value in zip(node.input, [input_data])
]
output_tensors = [
make_tensor_value_info(node.output[0], onnx.TensorProto.COMPLEX128,
input_data.shape)
] # type: ignore
graph = make_graph([node], "compute_graph", input_tensors, output_tensors)
model = make_model(graph, producer_name="NgraphBackend")
with pytest.raises(RuntimeError):
import_onnx_model(model)
def test_nodes_topologically_sorted():
# test analysis pass (nodes_topologically_sorted) with different models
# test with data/onnx/mnist-conv/model.onnx
raw_m = get_data("finn.data", "onnx/mnist-conv/model.onnx")
model = ModelWrapper(raw_m)
ret = model.analysis(ta.nodes_topologically_sorted)
assert ret["nodes_topologically_sorted"] is True
# remove first node and add it at the end
graph = model.graph
first_node = graph.node[0]
graph.node.remove(first_node)
graph.node.append(first_node)
ret = model.analysis(ta.nodes_topologically_sorted)
assert ret["nodes_topologically_sorted"] is False
# test with manually created small network
Neg_node = oh.make_node("Neg", inputs=["in1"], outputs=["neg1"])
Round_node = oh.make_node("Round", inputs=["neg1"], outputs=["round1"])
Ceil_node = oh.make_node("Ceil", inputs=["neg1"], outputs=["ceil1"])
Add_node = oh.make_node("Add",
inputs=["round1", "ceil1"],
outputs=["out1"])
in1 = oh.make_tensor_value_info("in1", TensorProto.FLOAT, [4, 4])
out1 = oh.make_tensor_value_info("out1", TensorProto.FLOAT, [4, 4])
graph = oh.make_graph(
nodes=[Neg_node, Round_node, Ceil_node, Add_node],
name="simple_graph",
inputs=[in1],
outputs=[out1],
value_info=[
oh.make_tensor_value_info("neg1", TensorProto.FLOAT, [4, 4]),
oh.make_tensor_value_info("round1", TensorProto.FLOAT, [4, 4]),
oh.make_tensor_value_info("ceil1", TensorProto.FLOAT, [4, 4]),
],
)
onnx_model = oh.make_model(graph, producer_name="simple-model")
model = ModelWrapper(onnx_model)
ret = model.analysis(ta.nodes_topologically_sorted)
assert ret["nodes_topologically_sorted"] is True
# create same graph but with "wrong" node order
graph = oh.make_graph(
nodes=[Round_node, Ceil_node, Neg_node, Add_node],
name="simple_graph",
inputs=[in1],
outputs=[out1],
value_info=[
oh.make_tensor_value_info("neg1", TensorProto.FLOAT, [4, 4]),
oh.make_tensor_value_info("round1", TensorProto.FLOAT, [4, 4]),
oh.make_tensor_value_info("ceil1", TensorProto.FLOAT, [4, 4]),
],
)
onnx_model = oh.make_model(graph, producer_name="simple-model")
model = ModelWrapper(onnx_model)
ret = model.analysis(ta.nodes_topologically_sorted)
assert ret["nodes_topologically_sorted"] is False
def test_gemm_conversion(self):
node_def = make_node('Gemm', ['A', 'B', 'C'], ["Y"], alpha=2., beta=3.)
node_def_broadcast = make_node('Gemm', ['A', 'B', 'C'], ["Y"],
alpha=2.,
beta=3.,
broadcast=1)
node_def_transpose_b = make_node('Gemm', ['A', 'B', 'C'], ["Y"],
alpha=2.,
beta=3.,
transB=1)
node_def_transpose_b_broadcast = make_node('Gemm', ['A', 'B', 'C'],
["Y"],
alpha=2.,
beta=3.,
transB=1,
broadcast=1)
backend = C.Caffe2Backend()
# without broadcast and without shape info, gemm will be
# converted to matmul + add
_, op_strs = backend.convert_node(node_def.SerializeToString())
op_names = []
for s in op_strs:
op = caffe2_pb2.OperatorDef()
op.ParseFromString(s)
op_names.append(op.type)
self.assertEqual(op_names, ['Scale', 'Scale', 'MatMul', 'Add'])
# opset7
# If C is a 1d tensor, gemm will be converted to FC/FCTransposed
_, op_strs = backend.convert_node(
node_def_transpose_b.SerializeToString(), [
make_tensor_value_info("C", onnx.TensorProto.FLOAT,
(3, )).SerializeToString()
], 7)
op_names = []
for s in op_strs:
op = caffe2_pb2.OperatorDef()
op.ParseFromString(s)
op_names.append(op.type)
self.assertEqual(op_names, ['Scale', 'Scale', 'FC'])
_, op_strs = backend.convert_node(node_def.SerializeToString(), [
make_tensor_value_info("C", onnx.TensorProto.FLOAT,
(3, )).SerializeToString()
], 7)
op_names = []
for s in op_strs:
op = caffe2_pb2.OperatorDef()
op.ParseFromString(s)
op_names.append(op.type)
self.assertEqual(op_names, ['Scale', 'Scale', 'FCTransposed'])
# opset6 without broadcast(C should match A*B's dim)
# The gemm will be converted to matmul + add, since the FC requires c
# to be 1d tensor.
_, op_strs = backend.convert_node(node_def.SerializeToString(), [
make_tensor_value_info("A", onnx.TensorProto.FLOAT,
(3, 2)).SerializeToString(),
make_tensor_value_info("B", onnx.TensorProto.FLOAT,
(2, 3)).SerializeToString(),
make_tensor_value_info("C", onnx.TensorProto.FLOAT,
(3, 3)).SerializeToString()
], 6)
op_names = []
for s in op_strs:
op = caffe2_pb2.OperatorDef()
op.ParseFromString(s)
op_names.append(op.type)
self.assertEqual(op_names, ['Scale', 'Scale', 'MatMul', 'Add'])
# opset6 with broadcast
# If C is a 1d tensor, gemm will be converted to FC/FCTransposed
_, op_strs = backend.convert_node(
node_def_transpose_b_broadcast.SerializeToString(), [
make_tensor_value_info("C", onnx.TensorProto.FLOAT,
(3, )).SerializeToString()
], 6)
op_names = []
for s in op_strs:
op = caffe2_pb2.OperatorDef()
op.ParseFromString(s)
op_names.append(op.type)
self.assertEqual(op_names, ['Scale', 'Scale', 'FC'])
_, op_strs = backend.convert_node(
node_def_broadcast.SerializeToString(), [
make_tensor_value_info("C", onnx.TensorProto.FLOAT,
(3, )).SerializeToString()
], 6)
op_names = []
for s in op_strs:
op = caffe2_pb2.OperatorDef()
op.ParseFromString(s)
op_names.append(op.type)
self.assertEqual(op_names, ['Scale', 'Scale', 'FCTransposed'])
# opset7
# If C is a scalar and B's last dim is 1, gemm will be converted to FC/FCTransposed
_, op_strs = backend.convert_node(
node_def_transpose_b.SerializeToString(), [
make_tensor_value_info("B", onnx.TensorProto.FLOAT,
(1, 2)).SerializeToString(),
make_tensor_value_info("C", onnx.TensorProto.FLOAT,
(1, )).SerializeToString()
], 7)
op_names = []
for s in op_strs:
op = caffe2_pb2.OperatorDef()
op.ParseFromString(s)
op_names.append(op.type)
self.assertEqual(op_names, ['Scale', 'Scale', 'FC'])
_, op_strs = backend.convert_node(node_def.SerializeToString(), [
make_tensor_value_info("B", onnx.TensorProto.FLOAT,
(2, 1)).SerializeToString(),
make_tensor_value_info("C", onnx.TensorProto.FLOAT,
(1, )).SerializeToString()
], 7)
op_names = []
for s in op_strs:
op = caffe2_pb2.OperatorDef()
op.ParseFromString(s)
op_names.append(op.type)
self.assertEqual(op_names, ['Scale', 'Scale', 'FCTransposed'])
# If C is a scalar and B's last dim is not 1, gemm will be converted
# to matmul + add.
_, op_strs = backend.convert_node(
node_def_transpose_b.SerializeToString(), [
make_tensor_value_info("B", onnx.TensorProto.FLOAT,
(2, 2)).SerializeToString(),
make_tensor_value_info("C", onnx.TensorProto.FLOAT,
(1, )).SerializeToString()
], 7)
op_names = []
for s in op_strs:
op = caffe2_pb2.OperatorDef()
op.ParseFromString(s)
op_names.append(op.type)
self.assertEqual(op_names, ['Scale', 'Scale', 'MatMul', 'Add'])
# If C is a scalar and B's shape info is not available,
# gemm will be converted to matmul + add.
_, op_strs = backend.convert_node(
node_def_transpose_b.SerializeToString(), [
make_tensor_value_info("C", onnx.TensorProto.FLOAT,
(1, )).SerializeToString()
], 7)
op_names = []
for s in op_strs:
op = caffe2_pb2.OperatorDef()
op.ParseFromString(s)
op_names.append(op.type)
self.assertEqual(op_names, ['Scale', 'Scale', 'MatMul', 'Add'])
def GenerateModel6(model_name):
nodes = [ # LayerNorm subgraph
helper.make_node("Shape", ["input_ids"], ["shape1_out"], "shape1"),
helper.make_node("Gather", ["shape1_out", "indices_0"], ["gather0_out"], "gather0"),
helper.make_node("Unsqueeze", ["gather0_out", "axes_0"], ["unsqueeze0_out"], "unsqueeze0") if opset_version == 13 \
else helper.make_node("Unsqueeze", ["gather0_out"], ["unsqueeze0_out"], "unsqueeze0", axes=[0]),
helper.make_node("Shape", ["input_ids"], ["shape2_out"], "shape2"),
helper.make_node("Gather", ["shape2_out", "indices_1"], ["gather1_out"], "gather1"),
helper.make_node("Unsqueeze", ["gather1_out", "axes_0"], ["unsqueeze1_out"], "unsqueeze1") if opset_version == 13 \
else helper.make_node("Unsqueeze", ["gather1_out"], ["unsqueeze1_out"], "unsqueeze1", axes=[0]),
helper.make_node("Concat", ["unsqueeze0_out", "unsqueeze1_out"], ["concat_out"], "concat", axis=0),
helper.make_node("Reshape", ["concat_out", "reshape_init"], ["reshape_out"], "reshape"),
helper.make_node("Equal", ["reshape_out", "equal_init"], ["equal_out"], "equal"),
helper.make_node("Where", ["equal_out", "where_init", "reshape_out"], ["where_out"], "where"),
helper.make_node("Range", ["start_0", "gather1_out", "delta_1"], ["range_out"], "range"),
helper.make_node("Unsqueeze", ["range_out", "axes_0"], ["unsqueeze2_out"], "unsqueeze2") if opset_version == 13 \
else helper.make_node("Unsqueeze", ["range_out"], ["unsqueeze2_out"], "unsqueeze2", axes=[0]),
helper.make_node("Expand", ["unsqueeze2_out", "where_out"], ["expand_out"], "expand"),
helper.make_node("Gather", ["pos_embed", "expand_out"], ["pos_gather_out"], "pos_gather"),
helper.make_node("Gather", ["word_embed", "input_ids"], ["word_gather_out"], "word_gather"),
helper.make_node("Add", ["word_gather_out", "pos_gather_out"], ["word_add_pos_out"], "word_add_pos"),
helper.make_node("Gather", ["seg_embed", "segment_ids"], ["seg_gather_out"], "seg_gather"),
helper.make_node("Add", ["word_add_pos_out", "seg_gather_out"], ["add3_out"], "add3"),
helper.make_node("LayerNormalization", ["add3_out", "layer_norm_weight", "layer_norm_bias"], ["layernorm_out"],
"layernorm",
axis=-1,
epsion=0.000009999999747378752),
helper.make_node("Cast", ["input_mask"], ["mask_cast_out"], "mask_cast", to=6),
helper.make_node("ReduceSum", ["mask_cast_out", "axes_1"], ["mask_index_out"], "mask_index", keepdims=0) if opset_version == 13 \
else helper.make_node("ReduceSum", ["mask_cast_out"], ["mask_index_out"], "mask_index", axes=[1], keepdims=0),
helper.make_node("Attention", ["layernorm_out", "qkv_weights", "qkv_bias", "mask_index_out"], ["att_out"],
"att",
domain="com.microsoft",
num_heads=2),
helper.make_node("MatMul", ["att_out", "matmul_weight"], ["matmul_out"], "matmul"),
helper.make_node("Add", ["matmul_out", "add_bias"], ["add_out"], "add"),
helper.make_node("Add", ["add_out", "layernorm_out"], ["add2_out"], "add2")
]
# hidden_size=4, num_heads=2, max_seq_length=3
initializers = [ # initializers
helper.make_tensor('indices_0', TensorProto.INT64, [], [0]),
helper.make_tensor('indices_1', TensorProto.INT64, [], [1]),
helper.make_tensor('start_0', TensorProto.INT64, [], [0]),
helper.make_tensor('delta_1', TensorProto.INT64, [], [1]),
helper.make_tensor('word_embed', TensorProto.FLOAT, [2, 4], [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]),
helper.make_tensor('pos_embed', TensorProto.FLOAT, [4, 4],
[1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]),
helper.make_tensor('seg_embed', TensorProto.FLOAT, [2, 4], [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]),
helper.make_tensor('layer_norm_weight', TensorProto.FLOAT, [4], [1.0, 2.0, 3.0, 4.0]),
helper.make_tensor('layer_norm_bias', TensorProto.FLOAT, [4], [0.1, 0.2, 0.3, 0.4]),
helper.make_tensor('qkv_weights', TensorProto.FLOAT, [4, 12], [0.1] * 4 * 12),
helper.make_tensor('qkv_bias', TensorProto.FLOAT, [12], [0.1] * 12),
helper.make_tensor('matmul_weight', TensorProto.FLOAT, [4, 4],
[1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]),
helper.make_tensor('add_bias', TensorProto.FLOAT, [4], [0.1, 0.2, 0.3, 0.4]),
helper.make_tensor('reshape_init', TensorProto.INT64, [1], [-1]),
helper.make_tensor('equal_init', TensorProto.INT64, [2], [-1, -1]),
helper.make_tensor('where_init', TensorProto.INT64, [2], [1, 1]),
helper.make_tensor('axes_0', TensorProto.INT64, [1], [0]),
helper.make_tensor('axes_1', TensorProto.INT64, [1], [1]),
]
graph = helper.make_graph(
nodes,
"EmbedLayerNorm_format6", #name
[ # inputs
helper.make_tensor_value_info('input_ids', TensorProto.INT64, ['batch', 3]),
helper.make_tensor_value_info('segment_ids', TensorProto.INT64, ['batch', 3]),
helper.make_tensor_value_info('input_mask', TensorProto.INT64, ['batch', 3]),
],
[ # outputs
helper.make_tensor_value_info('add2_out', TensorProto.FLOAT, ['batch', 3, 4]),
],
initializers)
model = helper.make_model(graph)
onnx.save(model, model_name)
def GenerateModel5(model_name):
batch_size = 2
hidden_size = 4
attention_heads = 2
sequence_length = 3
nodes = [
helper.make_node("Gather", ["word_embed", "input_ids"], ["word_gather_out"], "word_gather", axis=0),
helper.make_node("Add", ["word_gather_out", "pos_gather_out"], ["word_add_pos_out"], "word_add_pos"),
helper.make_node("Gather", ["seg_embed", "segment_ids"], ["seg_gather_out"], "seg_gather", axis=0),
helper.make_node("Add", ["word_add_pos_out", "seg_gather_out"], ["add3_out"], "add3"),
helper.make_node("LayerNormalization", ["add3_out", "layer_norm_weight", "layer_norm_bias"], ["layernorm_out"],
"layernorm",
axis=-1,
epsion=0.000009999999747378752),
helper.make_node("Cast", ["input_mask"], ["mask_cast_out"], "mask_cast", to=6),
helper.make_node("ReduceSum", ["mask_cast_out", "axes_1"], ["mask_index_out"], "mask_index", keepdims=0) if opset_version == 13 \
else helper.make_node("ReduceSum", ["mask_cast_out"], ["mask_index_out"], "mask_index", axes=[1], keepdims=0),
helper.make_node("Attention", ["layernorm_out", "qkv_weights", "qkv_bias", "mask_index_out"], ["att_out"],
"att",
domain="com.microsoft",
num_heads=attention_heads),
helper.make_node("MatMul", ["att_out", "matmul_weight"], ["matmul_out"], "matmul"),
helper.make_node("Add", ["matmul_out", "add_bias"], ["add_out"], "add"),
helper.make_node("Add", ["add_out", "layernorm_out"], ["add2_out"], "add2")
]
qkv_weights = [1.0] * hidden_size * (3 * hidden_size)
initializers = [ # initializers
helper.make_tensor('word_embed', TensorProto.FLOAT, [2, hidden_size], [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]),
helper.make_tensor('pos_gather_out', TensorProto.FLOAT, [batch_size, sequence_length, hidden_size], [
1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 8.0, 7.0, 6.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0,
8.0, 7.0, 6.0
]),
helper.make_tensor('seg_embed', TensorProto.FLOAT, [2, hidden_size], [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]),
helper.make_tensor('layer_norm_weight', TensorProto.FLOAT, [hidden_size], [1.0, 2.0, 3.0, 4.0]),
helper.make_tensor('layer_norm_bias', TensorProto.FLOAT, [hidden_size], [0.1, 0.2, 0.3, 0.4]),
helper.make_tensor('qkv_weights', TensorProto.FLOAT, [hidden_size, 3 * hidden_size], qkv_weights),
helper.make_tensor('qkv_bias', TensorProto.FLOAT, [3 * hidden_size],
[0.1, 0.2, 0.3, 0.4, 0.1, 0.2, 0.3, 0.4, 0.1, 0.2, 0.3, 0.4]),
helper.make_tensor('matmul_weight', TensorProto.FLOAT, [hidden_size, hidden_size],
[1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]),
helper.make_tensor('add_bias', TensorProto.FLOAT, [hidden_size], [0.1, 0.2, 0.3, 0.4]),
helper.make_tensor('axes_1', TensorProto.INT64, [1], [1]),
]
graph = helper.make_graph(
nodes,
"EmbedLayerNorm_format5", #name
[ # inputs
helper.make_tensor_value_info('input_ids', TensorProto.INT64, [batch_size, sequence_length]),
helper.make_tensor_value_info('segment_ids', TensorProto.INT64, [batch_size, sequence_length]),
helper.make_tensor_value_info('input_mask', TensorProto.INT64, [batch_size, sequence_length]),
],
[ # outputs
helper.make_tensor_value_info('add2_out', TensorProto.FLOAT, [batch_size, sequence_length, hidden_size]),
],
initializers)
model = helper.make_model(graph)
onnx.save(model, model_name)
# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
import onnx
from onnx import helper, TensorProto
INPUT_1 = helper.make_tensor_value_info('input1', TensorProto.FLOAT, [1])
OUTPUT = helper.make_tensor_value_info('output', TensorProto.INT8, [1])
nodes = [
helper.make_node(
'Cast',
['input1'],
['output'],
to=TensorProto.INT8
),
]
graph_def = helper.make_graph(
nodes,
'cast',
[INPUT_1],
[OUTPUT],
)
model_def = helper.make_model(graph_def, producer_name='cast_float_int8.py', opset_imports=[onnx.OperatorSetIdProto(version=12)])
onnx.save(model_def, 'cast_float_int8.onnx')
inputs=['A_reshaped', 'mul'],
outputs=['equal'],
name='equal'),
helper.make_node(op_type="Where",
inputs=['equal', 'const1', 'A_reshaped'],
outputs=['where'],
name='where'),
helper.make_node(op_type="Expand",
inputs=['B', 'where'],
outputs=['C'],
name='expand'),
],
name='test-model',
inputs=[
# create inputs with symbolic dims
helper.make_tensor_value_info("A", TensorProto.FLOAT, None),
helper.make_tensor_value_info("B", TensorProto.FLOAT, None),
],
outputs=[helper.make_tensor_value_info('C', TensorProto.FLOAT, None)],
initializer=[
helper.make_tensor('shape', TensorProto.INT64, [1], [-1]),
helper.make_tensor('neg_one', TensorProto.INT64, [1], [-1]),
])
model = helper.make_model(graph_def_0,
opset_imports=[helper.make_operatorsetid("", 12)])
onnx.save_model(model, "cpu_fallback_pattern_0.onnx")
graph_def_1 = helper.make_graph(
nodes=[
helper.make_node(op_type="Shape",
# onnx_model = onnx.load("../convert/artosyn_paddle/artosyn_paddle_resnet_v2_50_onnx.onnx")
graph = onnx_model.graph
# rewrite the input tensor of graph
input_tensor = graph.input[0]
input_shape = input_tensor.type.tensor_type.shape.dim
input_tensor_new = onnx.helper.make_tensor_value_info(name = input_tensor.name, elem_type = 1,
shape = [1, input_shape[1].dim_value, input_shape[2].dim_value, input_shape[3].dim_value])
graph.input.remove(input_tensor)
graph.input.insert(0, input_tensor_new)
# append all tensor infos to graph input
weight_infos = []
tensors = graph.initializer
for i, tensor in enumerate(tensors):
value_info = helper.make_tensor_value_info(tensor.name, ONNX_DTYPE[tensor.data_type], tensor.dims)
weight_infos.append(value_info)
graph.input.insert(i+1, value_info) # because 0 is for placeholder, so start index is 1
node = graph.node
print("---------------------------------")
print(graph.input[0])
print(len(graph.input))
value_info = graph.value_info
print("Before shape inference: \n")
# print(value_info)
print("------------------------------------------------------------")
print("After shape inference: \n")
inferred_onnx_model = shape_inference.infer_shapes(onnx_model)
onnx.checker.check_model(onnx_model)
def create_global_net(self, shape, op, ir_version):
"""
ONNX net IR net
Input->GlobalPooling>Output => Input->Pooling
"""
#
# Create ONNX model
#
import onnx
from onnx import helper
from onnx import TensorProto
out_shape = np.ones(len(shape))
out_shape[:2] = np.array(shape)[:2]
out_shape = out_shape.astype(np.int).tolist()
input = helper.make_tensor_value_info('input', TensorProto.FLOAT,
shape)
output = helper.make_tensor_value_info('output', TensorProto.FLOAT,
out_shape)
node_def = onnx.helper.make_node(op,
inputs=['input'],
outputs=['output'])
# Create the graph (GraphProto)
graph_def = helper.make_graph(
[node_def],
'test_model',
[input],
[output],
)
# Create the model (ModelProto)
onnx_net = helper.make_model(graph_def, producer_name='test_model')
#
# Create reference IR net
#
ref_net = None
if check_ir_version(10, None, ir_version):
nodes_attributes = {
'input': {
'kind': 'op',
'type': 'Parameter'
},
'input_data': {
'shape': shape,
'kind': 'data'
},
'input_axes_data': {
'kind': 'data',
'value': list(range(2, len(shape)))
},
'axes': {
'kind': 'op',
'type': 'Const'
},
'axes_data': {
'shape': [len(shape) - 2],
'kind': 'data'
},
'node': {
'kind': 'op',
'type': None
},
'node_data': {
'shape': out_shape,
'kind': 'data'
},
'result': {
'kind': 'op',
'type': 'Result'
}
}
if op == 'GlobalAveragePool':
nodes_attributes['node']['type'] = 'ReduceMean'
else:
nodes_attributes['node']['type'] = 'ReduceMax'
ref_net = build_graph(nodes_attributes,
[('input', 'input_data'),
('input_data', 'node'),
('input_axes_data', 'axes'),
('axes', 'axes_data'),
('axes_data', 'node'),
('node', 'node_data'),
('node_data', 'result')])
return onnx_net, ref_net
import onnx
from onnx import helper
from onnx import TensorProto, GraphProto, OperatorSetIdProto
from onnx import numpy_helper
import numpy as np
X1 = helper.make_tensor_value_info('x1', TensorProto.INT64, [4, 4])
X2 = helper.make_tensor_value_info('x2', TensorProto.INT64, [4, 1])
X3 = helper.make_tensor_value_info('x3', TensorProto.INT64, [4, 1])
Y = helper.make_tensor_value_info('output', TensorProto.INT64, [4, 4])
less1 = helper.make_node('Less', ['x1', 'x2'], ['less1'], name='less1')
less2 = helper.make_node('Less', ['x1', 'x3'], ['less2'], name='less2')
cast1 = helper.make_node('Cast', ['less1'], ['cast1'], name='cast1', to=9)
and_node = helper.make_node('And', ['cast1', 'less2'], ['and_node'],
name='and_node')
cast2 = helper.make_node('Cast', ['and_node'], ['cast2'], name='cast2', to=9)
cast3 = helper.make_node('Cast', ['cast2'], ['cast3'], name='cast3', to=1)
cast4 = helper.make_node('Cast', ['x1'], ['cast4'], name='cast4', to=7)
cast5 = helper.make_node('Cast', ['cast4'], ['cast5'], name='cast5', to=1)
matmul = helper.make_node('MatMul', ['cast3', 'cast5'], ['matmul'],
name='matmul')
cast6 = helper.make_node('Cast', ['matmul'], ['cast6'], name='cast6', to=7)
cast7 = helper.make_node('Cast', ['cast6'], ['output'], name='cast7', to=7)
# Create the graph (GraphProto)
graph_def = helper.make_graph([
less1, less2, cast1, and_node, cast2, cast3, cast4, cast5, matmul, cast6,
cast7
], 'cast_elimination_model', [X1, X2, X3], [Y])
def create_net(self,
shape,
kernel_shape,
pads,
strides,
op,
ir_version,
count_include_pad=None,
auto_pad=None,
storage_order=None,
ceil=False,
opset=None):
"""
ONNX net IR net
Input->Pooling>Output => Input->Pooling
"""
#
# Create ONNX model
#
import onnx
from onnx import helper
from onnx import TensorProto
node_args = dict(kernel_shape=kernel_shape)
if auto_pad is not None:
node_args['auto_pad'] = auto_pad
if auto_pad == 'VALID':
pads = np.zeros(len(shape[2:]) * 2, dtype=np.int)
else:
auto_pad = 'NOTSET'
if count_include_pad is not None:
node_args['count_include_pad'] = count_include_pad
else:
count_include_pad = 0
if storage_order is not None:
node_args['storage_order'] = storage_order
if pads is not None:
if auto_pad == 'NOTSET':
node_args['pads'] = pads
_pads = np.transpose(np.array(pads).reshape([2, -1]))
else:
_pads = np.zeros([len(kernel_shape), 2])
if strides is not None:
node_args['strides'] = strides
else:
strides = np.ones(len(kernel_shape))
if ceil:
node_args['ceil_mode'] = 1
if auto_pad in ['SAME_UPPER', 'SAME_LOWER']:
out_spacial_shape = np.ceil(
np.array(shape[2:], dtype=np.float) / strides)
else:
rounding = np.ceil if ceil else np.floor
out_spacial_shape = rounding(
(float_array(shape[2:]) + np.add(_pads[:, 0], _pads[:, 1]) -
float_array(kernel_shape)) / strides + 1)
out_shape = np.array(shape)
out_shape[2:] = out_spacial_shape
out_shape = out_shape.astype(np.int).tolist()
concat_axis = 0
out_concat_shape = out_shape.copy()
out_concat_shape[concat_axis] *= 2
input = helper.make_tensor_value_info('input', TensorProto.FLOAT,
shape)
output = helper.make_tensor_value_info('output', TensorProto.FLOAT,
out_concat_shape)
constant = np.random.randint(-127, 127, out_shape).astype(np.float)
node_def = onnx.helper.make_node(op,
inputs=['input'],
outputs=['pool'],
**node_args)
node_const_def = onnx.helper.make_node(
'Constant',
inputs=[],
outputs=['const1'],
value=helper.make_tensor(
name='const_tensor',
data_type=TensorProto.FLOAT,
dims=constant.shape,
vals=constant.flatten(),
),
)
node_concat_def = onnx.helper.make_node('Concat',
inputs=['pool', 'const1'],
outputs=['output'],
axis=concat_axis)
graph_def = helper.make_graph(
[node_def, node_const_def, node_concat_def],
'test_model',
[input],
[output],
)
# Create the model (ModelProto)
args = dict(producer_name='test_model')
if opset:
args['opset_imports'] = [helper.make_opsetid("", opset)]
onnx_net = helper.make_model(graph_def, **args)
#
# Create reference IR net
#
ref_net = None
if check_ir_version(10, None, ir_version):
nodes_attributes = {
'input': {
'kind': 'op',
'type': 'Parameter'
},
'input_data': {
'shape': shape,
'kind': 'data'
},
'node': {
'kind': 'op',
'type': None,
'pads_begin': _pads[:, 0] if len(shape) > 3 else _pads[0,
0],
'pads_end': _pads[:, 1] if len(shape) > 3 else _pads[0, 1],
'kernel': kernel_shape[0]
if len(kernel_shape) == 1 else kernel_shape,
'rounding_type':
'ceil' if auto_pad != 'NOTSET' or ceil else 'floor',
'auto_pad': None
},
'node_data': {
'shape': out_shape,
'kind': 'data'
},
'node_indicies_data': {
'shape': out_shape,
'kind': 'data'
},
'input_const_data': {
'kind': 'data',
'value': constant.flatten()
},
'const': {
'kind': 'op',
'type': 'Const'
},
'const_data': {
'shape': out_shape,
'kind': 'data'
},
'concat': {
'kind': 'op',
'type': 'Concat',
'axis': concat_axis
},
'concat_data': {
'shape': out_concat_shape,
'kind': 'data'
},
'result': {
'kind': 'op',
'type': 'Result'
}
}
if op == 'AveragePool':
nodes_attributes['node']['type'] = 'AvgPool'
nodes_attributes['node'][
'exclude-pad'] = True if count_include_pad == 0 else False
else:
nodes_attributes['node']['type'] = 'MaxPool'
edges = [('input', 'input_data'), ('input_data', 'node'),
('node', 'node_data', {
'out': 0
}), ('input_const_data', 'const'),
('const', 'const_data'), ('node_data', 'concat'),
('const_data', 'concat'), ('concat', 'concat_data'),
('concat_data', 'result')]
if op == "MaxPool":
edges.append(('node', 'node_indicies_data', {'out': 1}))
ref_net = build_graph(nodes_attributes,
edges,
nodes_with_edges_only=True)
return onnx_net, ref_net
def test_augment_graph(self):
''' TEST_CONFIG_1'''
# Conv
# |
# Clip
# |
# MatMul
A = helper.make_tensor_value_info('A', TensorProto.FLOAT, [1, 1, 5, 5])
B = helper.make_tensor_value_info('B', TensorProto.FLOAT, [1, 1, 3, 3])
E = helper.make_tensor_value_info('E', TensorProto.FLOAT, [1, 1, 5, 1])
F = helper.make_tensor_value_info('F', TensorProto.FLOAT, [1, 1, 5, 1])
conv_node = onnx.helper.make_node('Conv', ['A', 'B'], ['C'],
name='Conv',
kernel_shape=[3, 3],
pads=[1, 1, 1, 1])
clip_node = onnx.helper.make_node('Clip', ['C'], ['D'], name='Clip')
matmul_node = onnx.helper.make_node('MatMul', ['D', 'E'], ['F'],
name='MatMul')
graph = helper.make_graph([conv_node, clip_node, matmul_node],
'test_graph_1', [A, B, E], [F])
model = helper.make_model(graph)
test_model_path = './test_model_1.onnx'
onnx.save(model, test_model_path)
# Augmenting graph
data_reader = TestDataReader()
augmented_model_path = './augmented_test_model_1.onnx'
calibrater = ONNXCalibrater(test_model_path, data_reader,
['Conv', 'MatMul'], [], [],
augmented_model_path)
augmented_model = calibrater.augment_graph()
onnx.save(augmented_model, augmented_model_path)
# Checking if each added ReduceMin and ReduceMax node and its output exists
augmented_model_node_names = [
node.name for node in augmented_model.graph.node
]
augmented_model_outputs = [
output.name for output in augmented_model.graph.output
]
added_node_names = [
'C_ReduceMin', 'C_ReduceMax', 'D_ReduceMin', 'D_ReduceMax',
'F_ReduceMin', 'F_ReduceMax'
]
added_outputs = [
'C_ReduceMin', 'C_ReduceMax', 'D_ReduceMin', 'D_ReduceMax',
'F_ReduceMin', 'F_ReduceMax'
]
# Original 3 nodes + added ReduceMin/Max nodes * 6 (exlude graph input/output)
self.assertEqual(len(augmented_model_node_names), 9)
# Original 1 graph output + added outputs * 6
self.assertEqual(len(augmented_model_outputs), 7)
for name in added_node_names:
self.assertTrue(name in augmented_model_node_names)
for output in added_outputs:
self.assertTrue(output in augmented_model_outputs)
print('Finished TEST_CONFIG_1')
'''TEST_CONFIG_2'''
# Conv
# |
# Conv
G = helper.make_tensor_value_info('G', TensorProto.FLOAT, [1, 1, 5, 5])
H = helper.make_tensor_value_info('H', TensorProto.FLOAT, [1, 1, 3, 3])
J = helper.make_tensor_value_info('J', TensorProto.FLOAT, [1, 1, 3, 3])
K = helper.make_tensor_value_info('K', TensorProto.FLOAT, [1, 1, 5, 5])
conv_node_1 = onnx.helper.make_node('Conv', ['G', 'H'], ['I'],
name='Conv',
kernel_shape=[3, 3],
pads=[1, 1, 1, 1])
conv_node_2 = onnx.helper.make_node('Conv', ['I', 'J'], ['K'],
name='Conv',
kernel_shape=[3, 3],
pads=[1, 1, 1, 1])
graph = helper.make_graph([conv_node_1, conv_node_2], 'test_graph_2',
[G, H, J], [K])
model = helper.make_model(graph)
test_model_path = './test_model_2.onnx'
onnx.save(model, test_model_path)
# Augmenting graph
data_reader = TestDataReader()
augmented_model_path = './augmented_test_model_2.onnx'
calibrater = ONNXCalibrater(test_model_path, data_reader,
['Conv', 'MatMul'], [], [],
augmented_model_path)
augmented_model = calibrater.augment_graph()
onnx.save(augmented_model, augmented_model_path)
augmented_model_node_names = [
node.name for node in augmented_model.graph.node
]
augmented_model_outputs = [
output.name for output in augmented_model.graph.output
]
added_node_names = [
'I_ReduceMin', 'I_ReduceMax', 'K_ReduceMin', 'K_ReduceMax'
]
added_outputs = [
'I_ReduceMin', 'I_ReduceMax', 'K_ReduceMin', 'K_ReduceMax'
]
# Original 2 nodes + added ReduceMin/Max nodes * 4
self.assertEqual(len(augmented_model_node_names), 6)
# Original 1 graph output + added outputs * 4
self.assertEqual(len(augmented_model_outputs), 5)
for name in added_node_names:
self.assertTrue(name in augmented_model_node_names)
for output in added_outputs:
self.assertTrue(output in augmented_model_outputs)
print('Finished TEST_CONFIG_2')
'''TEST_CONFIG_3'''
# Relu
# |
# Conv \
# | |
# Clip |
# | /
# MatMul
L = helper.make_tensor_value_info('L', TensorProto.FLOAT, [1, 1, 5, 5])
N = helper.make_tensor_value_info('N', TensorProto.FLOAT, [1, 1, 3, 3])
Q = helper.make_tensor_value_info('Q', TensorProto.FLOAT, [1, 1, 5, 5])
relu_node = onnx.helper.make_node('Relu', ['L'], ['M'], name='Relu')
conv_node = onnx.helper.make_node('Conv', ['M', 'N'], ['O'],
name='Conv',
kernel_shape=[3, 3],
pads=[1, 1, 1, 1])
clip_node = onnx.helper.make_node('Clip', ['O'], ['P'], name='Clip')
matmul_node = onnx.helper.make_node('MatMul', ['P', 'M'], ['Q'],
name='MatMul')
graph = helper.make_graph(
[relu_node, conv_node, clip_node, matmul_node], 'test_graph_3',
[L, N], [Q])
model = helper.make_model(graph)
test_model_path = './test_model_3.onnx'
onnx.save(model, test_model_path)
# Augmenting graph
data_reader = TestDataReader()
augmented_model_path = './augmented_test_model_3.onnx'
calibrater = ONNXCalibrater(test_model_path, data_reader,
['Conv', 'MatMul'], [], [],
augmented_model_path)
augmented_model = calibrater.augment_graph()
onnx.save(augmented_model, augmented_model_path)
augmented_model_node_names = [
node.name for node in augmented_model.graph.node
]
augmented_model_outputs = [
output.name for output in augmented_model.graph.output
]
added_node_names = [
'M_ReduceMin', 'M_ReduceMax', 'O_ReduceMin', 'O_ReduceMax',
'P_ReduceMin', 'P_ReduceMax', 'Q_ReduceMin', 'Q_ReduceMax'
]
added_outputs = [
'M_ReduceMin', 'M_ReduceMax', 'O_ReduceMin', 'O_ReduceMax',
'P_ReduceMin', 'P_ReduceMax', 'Q_ReduceMin', 'Q_ReduceMax'
]
# Original 4 nodes + added ReduceMin/Max nodes * 8
self.assertEqual(len(augmented_model_node_names), 12)
# Original 1 graph output + added outputs * 8
self.assertEqual(len(augmented_model_outputs), 9)
for name in added_node_names:
self.assertTrue(name in augmented_model_node_names)
for output in added_outputs:
self.assertTrue(output in augmented_model_outputs)
print('Finished TEST_CONFIG_3')
def create_reduce_lp_const(self, shape, axes, keep_dims, reduce_p, ir_version):
"""
ONNX net IR net
Input->ReduceLX(axes)->Output => Input->ReduceLX
"""
#
# Create ONNX model
#
import onnx
from onnx import helper
from onnx import TensorProto
output_shape = shape.copy()
_axes = axes.copy() if axes is not None else list(range(len(shape)))
for axis in _axes:
output_shape[axis] = 1
if not keep_dims:
output_shape = [dim for dim in output_shape if dim != 1]
if len(output_shape) == 0:
output_shape = [1]
concat_axis = 0
concat_output_shape = output_shape.copy()
concat_output_shape[concat_axis] *= 2
input = helper.make_tensor_value_info('input', TensorProto.FLOAT, output_shape)
output = helper.make_tensor_value_info('output', TensorProto.FLOAT, concat_output_shape)
constant = np.random.randn(*shape).astype(np.float)
node_const_def = onnx.helper.make_node(
'Constant',
inputs=[],
outputs=['const1'],
value=helper.make_tensor(
name='const_tensor',
data_type=TensorProto.FLOAT,
dims=constant.shape,
vals=constant.flatten(),
),
)
args = dict(keepdims=keep_dims)
if axes:
args['axes'] = axes
node_def = onnx.helper.make_node(
"ReduceL" + str(reduce_p),
inputs=['const1'],
outputs=['reduce'],
**args
)
node_concat_def = onnx.helper.make_node(
'Concat',
inputs=['input', 'reduce'],
outputs=['output'],
axis=concat_axis
)
# Create the graph (GraphProto)
graph_def = helper.make_graph(
[node_const_def, node_def, node_concat_def],
'test_model',
[input],
[output],
)
# Create the model (ModelProto)
onnx_net = helper.make_model(graph_def, producer_name='test_model')
#
# Create reference IR net
# Please, specify 'type': 'Input' for input node
# Moreover, do not forget to validate ALL layer attributes!!!
#
constant = np.power(np.sum(a=np.abs(np.power(constant, reduce_p)), axis=tuple(_axes), keepdims=keep_dims), 1 / reduce_p)
ref_net = None
if check_ir_version(10, None, ir_version):
nodes_attributes = {
'input': {'kind': 'op', 'type': 'Parameter'},
'input_data': {'shape': output_shape, 'kind': 'data'},
'input_const_data': {'kind': 'data', 'value': constant.flatten()},
'const': {'kind': 'op', 'type': 'Const'},
'const_data': {'shape': constant.shape, 'kind': 'data'},
'concat': {'kind': 'op', 'type': 'Concat', 'axis': concat_axis},
'concat_data': {'shape': concat_output_shape, 'kind': 'data'},
'result': {'kind': 'op', 'type': 'Result'}
}
ref_net = build_graph(nodes_attributes,
[('input', 'input_data'),
('input_const_data', 'const'),
('const', 'const_data'),
('input_data', 'concat'),
('const_data', 'concat'),
('concat', 'concat_data'),
('concat_data', 'result')
])
return onnx_net, ref_net
def test_quant_param_calculation(self):
'''TEST_CONFIG_4'''
# Relu
# | \
# Conv \
# | \
# Relu |
# | Conv
# Conv /
# \ /
# |
# Add
input0 = helper.make_tensor_value_info('input0', TensorProto.FLOAT,
[1, 3, 1, 3])
output = helper.make_tensor_value_info('output', TensorProto.FLOAT,
[1, 3, 1, 3])
X1_weight = generate_input_initializer([3, 3, 1, 1], np.float32,
'X1_weight')
X1_bias = generate_input_initializer([3], np.float32, 'X1_bias')
X3_weight = generate_input_initializer([3, 3, 1, 1], np.float32,
'X3_weight')
X3_bias = generate_input_initializer([3], np.float32, 'X3_bias')
X5_weight = generate_input_initializer([3, 3, 1, 1], np.float32,
'X5_weight')
X5_bias = generate_input_initializer([3], np.float32, 'X5_bias')
relu_node_1 = onnx.helper.make_node('Relu', ['input0'], ['X1'],
name='Relu1')
conv_node_1 = onnx.helper.make_node('Conv',
['X1', 'X1_weight', 'X1_bias'],
['X2'],
name='Conv1')
relu_node_2 = onnx.helper.make_node('Relu', ['X2'], ['X3'],
name='Relu2')
conv_node_2 = onnx.helper.make_node('Conv',
['X3', 'X3_weight', 'X3_bias'],
['X4'],
name='Conv2')
conv_node_3 = onnx.helper.make_node('Conv',
['X1', 'X5_weight', 'X5_bias'],
['X5'],
name='Conv3')
add_node = onnx.helper.make_node('Add', ['X4', 'X5'], ['output'],
name='Add')
graph = helper.make_graph([
relu_node_1, conv_node_1, relu_node_2, conv_node_2, conv_node_3,
add_node
], 'test_graph_4', [input0], [output])
graph.initializer.add().CopyFrom(X1_weight)
graph.initializer.add().CopyFrom(X1_bias)
graph.initializer.add().CopyFrom(X3_weight)
graph.initializer.add().CopyFrom(X3_bias)
graph.initializer.add().CopyFrom(X5_weight)
graph.initializer.add().CopyFrom(X5_bias)
model = helper.make_model(graph)
test_model_path = './test_model_4.onnx'
onnx.save(model, test_model_path)
data_reader = TestDataReaderSecond()
augmented_model_path = './augmented_test_model_4.onnx'
calibrater = ONNXCalibrater(test_model_path, data_reader,
['Conv', 'MatMul'], [], [],
augmented_model_path)
augmented_model = calibrater.augment_graph()
onnx.save(augmented_model, augmented_model_path)
#test calculation of quantization params
#TO_DO: check rmin/rmax
dict_for_quantization = calibrater.get_intermediate_outputs()
quantization_params_dict = calibrater.calculate_quantization_params(
dict_for_quantization)
#check the size of the quantization dictionary
self.assertEqual(len(quantization_params_dict), 5)
#check the computation of zp and scale
for key, value in quantization_params_dict.items():
self.assertTrue(value is not None)
self.assertTrue(len(value) == 2)
thresholds = dict_for_quantization[key]
rmin = min(thresholds[0], 0)
rmax = max(thresholds[1], 0)
if key == 'X2': #next_node is Relu
if rmin < 0: rmin = 0
scale_expected = np.float32((rmax - rmin) /
255 if rmin != rmax else 1)
zp_expected = np.uint8(
round(max(0, min(255, (0 - rmin) / scale_expected))))
zp_actual = value[0]
scale_actual = value[1]
self.assertEqual(zp_expected, zp_actual)
self.assertEqual(scale_expected, scale_actual)
print('Finished' + ' test calculation of quantization params.')
# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
import onnx
from onnx import helper, TensorProto
IN8 = helper.make_tensor_value_info('in8', TensorProto.INT8, [3])
IN16 = helper.make_tensor_value_info('in16', TensorProto.BFLOAT16, [3])
OUT8 = helper.make_tensor_value_info('out8', TensorProto.INT8, [3])
OUT16 = helper.make_tensor_value_info('out16', TensorProto.BFLOAT16, [3])
nodes = [
helper.make_node(
'Cast',
['in8'],
['out16'],
to=getattr(TensorProto, 'BFLOAT16')
),
helper.make_node(
'Cast',
['in16'],
['out8'],
to=getattr(TensorProto, 'INT8')
),
]
graph_def = helper.make_graph(
nodes,
'unstable_types',
[IN8, IN16],
[OUT8, OUT16],
)
model_def = helper.make_model(graph_def, producer_name='unstable_types.py', opset_imports=[onnx.OperatorSetIdProto(version=13)])
onnx.save(model_def, 'unstable_types.onnx')
def create_net(self, shape, ir_version):
"""
ONNX net IR net
Input->Sqrt->Output => Input->Power
"""
#
# Create ONNX model
#
import onnx
from onnx import helper
from onnx import TensorProto
input = helper.make_tensor_value_info('input', TensorProto.FLOAT,
shape)
output = helper.make_tensor_value_info('output', TensorProto.FLOAT,
shape)
node_def = onnx.helper.make_node('Sqrt',
inputs=['input'],
outputs=['output'])
# Create the graph (GraphProto)
graph_def = helper.make_graph(
[node_def],
'test_model',
[input],
[output],
)
# Create the model (ModelProto)
onnx_net = helper.make_model(graph_def, producer_name='test_model')
#
# Create reference IR net
#
ref_net = None
if check_ir_version(10, None, ir_version):
nodes_attributes = {
'input': {
'kind': 'op',
'type': 'Parameter'
},
'input_data': {
'shape': shape,
'kind': 'data'
},
'const_indata': {
'shape': None,
'kind': 'data'
},
'const': {
'kind': 'op',
'type': 'Const'
},
'const_data': {
'shape': np.ones(len(shape)),
'kind': 'data'
},
'node': {
'kind': 'op',
'type': 'Power'
},
'node_data': {
'shape': shape,
'kind': 'data'
},
'result': {
'kind': 'op',
'type': 'Result'
}
}
ref_net = build_graph(nodes_attributes, [('input', 'input_data'),
('const_indata', 'const'),
('const', 'const_data'),
('input_data', 'node'),
('const_data', 'node'),
('node', 'node_data'),
('node_data', 'result')])
return onnx_net, ref_net
def graph_def_to_onnx_graph(
cls,
graph_def,
init_func=None,
constants=None,
value_info=None,
graph_name=None,
verbose=True,
enforce_no_running=False,
):
if value_info is None: value_info = {}
if not isinstance(value_info, dict):
raise ValueError('Please pass value_info as a '
'name -> (type, shape) dictionary')
leaf_tensors = extract_leaf_tensors(graph_def)
initializer = extract_initializer(graph_def)
# Check whether we have got type shape info of all input
missing = (leaf_tensors - set(value_info.keys()) - initializer)
if missing:
raise RuntimeError(
'Could not find value info of inputs: {}'.format(
', '.join(missing)))
# Check if value_info contains the types/shapes of all the blobs, in
# which case we don't need to infer them by running the net.
run_native_graph = False
for op in graph_def.op:
for name in itertools.chain(op.input, op.output):
if name not in value_info:
run_native_graph = True
break
ws = None
# Get the value info of outputs and initializer
if run_native_graph and not enforce_no_running:
inputs = {}
for name, (elem_type, shape) in value_info.items():
inputs[name] = np.random.randn(*shape).astype(
mapping.TENSOR_TYPE_TO_NP_TYPE[elem_type])
ws, outputs, initializer = native_run_graph(
graph_def, inputs, initializer, init_func)
if enforce_no_running:
# In some cases(e.g. PyTorch), we had ran the graph
# outputs had been in ``value_info`` already
import dragon.core.workspace as ws
initializer = fetch_initializer(initializer)
# Prepare to make the graph
onnx_graph = GraphProto()
onnx_graph.name = graph_name if graph_name else graph_def.name
# Initializer should also be included in the inputs
value_info.update(
{init.name: (init.data_type, init.dims)
for init in initializer})
# Add initializer
onnx_graph.initializer.extend(initializer)
# Add inputs
onnx_graph.input.extend(
make_tensor_value_info(name=name,
elem_type=value_info[name][0],
shape=value_info[name][1])
for name in leaf_tensors)
# Add outputs
onnx_graph.output.extend(
make_tensor_value_info(name=name,
elem_type=value_info[name][0],
shape=value_info[name][1])
for name in set(graph_def.output))
# Add constants
if constants is not None:
for k, v in constants.items():
onnx_graph.initializer.extend(
[numpy_helper.from_array(v, name=k)])
# Add nodes
shapes, ssa_names, ssa_outputs = {}, {}, defaultdict(int)
for op in graph_def.op:
# Get the shape of inputs and outputs
for name in itertools.chain(op.input, op.output):
if ws and ws.HasTensor(name):
blob = ws.FetchTensor(name)
if hasattr(blob, 'shape'):
shapes[name] = blob.shape
else:
shapes[name] = value_info[name][1]
# SSA rewritten
op, shapes, ssa_names, ssa_outputs = \
cls._ssa_rewrite(op, shapes, ssa_names, ssa_outputs)
# Try to translate op => nodes
nodes, const_tensors = get_nodes_def(op, shapes, ws)
# Directly convert outputs as const tensors if necessary
if None in nodes:
const_tensors = [
numpy_helper.from_array(ws.FetchTensor(name), name=name)
for name in op.output
]
else:
onnx_graph.node.extend(nodes)
# Add const tensors
if const_tensors is not None:
onnx_graph.initializer.extend(const_tensors)
onnx_graph.input.extend([
cls._extract_value_info(tensor) for tensor in const_tensors
])
if verbose: print(printable_graph(onnx_graph))
return onnx_graph
def create_net_const(self, shape, precision, ir_version):
"""
ONNX net IR net
Input->Concat(+sqrt const)->Output => Input->Concat(+const)
"""
#
# Create ONNX model
#
import onnx
from onnx import helper
from onnx import TensorProto
concat_axis = 0
output_shape = shape.copy()
output_shape[concat_axis] *= 2
input = helper.make_tensor_value_info('input', TensorProto.FLOAT,
shape)
output = helper.make_tensor_value_info('output', TensorProto.FLOAT,
output_shape)
constant = np.random.rand(*shape).astype(np.float) * 255
node_const_def = onnx.helper.make_node(
'Constant',
inputs=[],
outputs=['const'],
value=helper.make_tensor(
name='const_tensor',
data_type=TensorProto.FLOAT,
dims=constant.shape,
vals=constant.flatten(),
),
)
node_def = onnx.helper.make_node('Sqrt',
inputs=['const'],
outputs=['sqrt'])
node_concat_def = onnx.helper.make_node('Concat',
inputs=['input', 'sqrt'],
outputs=['output'],
axis=concat_axis)
# Create the graph (GraphProto)
graph_def = helper.make_graph(
[node_const_def, node_def, node_concat_def],
'test_model',
[input],
[output],
)
# Create the model (ModelProto)
onnx_net = helper.make_model(graph_def, producer_name='test_model')
#
# Create reference IR net
#
constant = np.sqrt(constant)
if precision == 'FP16':
constant = constant.astype(np.float16)
ref_net = None
if check_ir_version(10, None, ir_version):
nodes_attributes = {
'input': {
'kind': 'op',
'type': 'Parameter'
},
'input_data': {
'shape': shape,
'kind': 'data'
},
'input_const_data': {
'kind': 'data',
'value': constant.flatten()
},
'const': {
'kind': 'op',
'type': 'Const'
},
'const_data': {
'shape': shape,
'kind': 'data'
},
'concat': {
'kind': 'op',
'type': 'Concat',
'axis': concat_axis
},
'concat_data': {
'shape': output_shape,
'kind': 'data'
},
'result': {
'kind': 'op',
'type': 'Result'
}
}
ref_net = build_graph(nodes_attributes,
[('input', 'input_data'),
('input_const_data', 'const'),
('const', 'const_data'),
('input_data', 'concat'),
('const_data', 'concat'),
('concat', 'concat_data'),
('concat_data', 'result')])
return onnx_net, ref_net
def _extract_value_info(tensor):
return make_tensor_value_info(name=tensor.name,
elem_type=tensor.data_type,
shape=tensor.dims)
def test_slice(self):
# test case 1 with normal inputs
axes = [0, 1, 2]
starts = [0, 0, 0]
ends = [2, 2, 2]
if legacy_opset_pre_ver(10):
node_def = helper.make_node("Slice", ["X"], ["S"],
axes=axes,
starts=starts,
ends=ends)
graph_def = helper.make_graph(
[node_def],
name="test_unknown_shape",
inputs=[
helper.make_tensor_value_info("X", TensorProto.FLOAT,
[None, None, None])
],
outputs=[
helper.make_tensor_value_info("S", TensorProto.FLOAT,
[None, None, None])
])
else:
node_def = helper.make_node("Slice",
["X", "starts", "ends", "axes"], ["S"])
graph_def = helper.make_graph(
[node_def],
name="test_unknown_shape",
inputs=[
helper.make_tensor_value_info("X", TensorProto.FLOAT,
[None, None, None]),
helper.make_tensor_value_info("starts", TensorProto.INT32,
[None]),
helper.make_tensor_value_info("ends", TensorProto.INT32,
[None]),
helper.make_tensor_value_info("axes", TensorProto.INT32,
[None]),
],
outputs=[
helper.make_tensor_value_info("S", TensorProto.FLOAT,
[None, None, None])
])
tf_rep = onnx_graph_to_tensorflow_rep(graph_def)
if legacy_opset_pre_ver(10):
x = self._get_rnd_float32(shape=[1000]).reshape([10, 10, 10])
output = tf_rep.run({"X": x})
np.testing.assert_almost_equal(output["S"], x[0:2, 0:2, 0:2])
else:
x = self._get_rnd_float32(shape=[1000]).reshape([10, 10, 10])
output = tf_rep.run({
"X": x,
"starts": starts,
"ends": ends,
"axes": axes
})
np.testing.assert_almost_equal(output["S"], x[0:2, 0:2, 0:2])
# test case 2 with negative, out-of-bound and default inputs
axes = [0, 2]
starts = [0, -7]
ends = [-8, 20]
steps = [1, 1]
if legacy_opset_pre_ver(10):
node_def = helper.make_node("Slice", ["X"], ["S"],
axes=axes,
starts=starts,
ends=ends)
graph_def = helper.make_graph(
[node_def],
name="test_unknown_shape",
inputs=[
helper.make_tensor_value_info("X", TensorProto.FLOAT,
[None, None, None])
],
outputs=[
helper.make_tensor_value_info("S", TensorProto.FLOAT,
[None, None, None])
])
else:
node_def = helper.make_node(
"Slice", ["X", "starts", "ends", "axes", "steps"], ["S"])
graph_def = helper.make_graph(
[node_def],
name="test_unknown_shape",
inputs=[
helper.make_tensor_value_info("X", TensorProto.FLOAT,
[None, None, None]),
helper.make_tensor_value_info("starts", TensorProto.INT32,
[None]),
helper.make_tensor_value_info("ends", TensorProto.INT32,
[None]),
helper.make_tensor_value_info("axes", TensorProto.INT32,
[None]),
helper.make_tensor_value_info("steps", TensorProto.INT32,
[None]),
],
outputs=[
helper.make_tensor_value_info("S", TensorProto.FLOAT,
[None, None, None])
])
tf_rep = onnx_graph_to_tensorflow_rep(graph_def)
if legacy_opset_pre_ver(10):
x = self._get_rnd_float32(shape=[1000]).reshape([10, 10, 10])
output = tf_rep.run({"X": x})
np.testing.assert_almost_equal(output["S"], x[0:-8, :, -7:20])
else:
x = self._get_rnd_float32(shape=[1000]).reshape([10, 10, 10])
output = tf_rep.run({
"X": x,
"starts": starts,
"ends": ends,
"axes": axes,
"steps": steps
})
np.testing.assert_almost_equal(output["S"], x[0:-8, :, -7:20])
# test case 3 with non-default steps
axes = [0, 1, 2]
starts = [0, 0, 0]
ends = [2, 2, 2]
steps = [2, -2, -1]
if not legacy_opset_pre_ver(10):
x = self._get_rnd_float32(shape=[1000]).reshape([10, 10, 10])
output = tf_rep.run({
"X": x,
"starts": starts,
"ends": ends,
"axes": axes,
"steps": steps
})
np.testing.assert_almost_equal(output["S"], x[0:2:2, 0:2:-2,
0:2:-1])
def create_net_const(self, shape, scale, precision, ir_version):
"""
ONNX net IR net
Input->Concat(+scaled const)->Output => Input->Concat(+const)
"""
#
# Create ONNX model
#
import onnx
from onnx import helper
from onnx import TensorProto
concat_axis = 0
output_shape = shape.copy()
output_shape[concat_axis] *= 2
input = helper.make_tensor_value_info('input', TensorProto.FLOAT,
shape)
output = helper.make_tensor_value_info('output', TensorProto.FLOAT,
output_shape)
constant = np.random.randint(-127, 127, shape).astype(np.float)
bias = np.random.randint(-10, 10, shape[1]).astype(np.float)
node_const_def = onnx.helper.make_node(
'Constant',
inputs=[],
outputs=['const1'],
value=helper.make_tensor(
name='const_tensor',
data_type=TensorProto.FLOAT,
dims=constant.shape,
vals=constant.flatten(),
),
)
node_def = onnx.helper.make_node('ImageScaler',
inputs=['const1'],
outputs=['scale'],
bias=bias,
scale=scale)
node_concat_def = onnx.helper.make_node('Concat',
inputs=['input', 'scale'],
outputs=['output'],
axis=concat_axis)
# Create the graph (GraphProto)
graph_def = helper.make_graph(
[node_const_def, node_def, node_concat_def],
'test_model',
[input],
[output],
)
# Create the model (ModelProto)
onnx_net = helper.make_model(graph_def, producer_name='test_model')
#
# Create reference IR net
#
ir_const = constant * scale + np.expand_dims(np.expand_dims([bias], 2),
3)
if precision == 'FP16':
ir_const = ir_const.astype(np.float16)
ref_net = None
return onnx_net, ref_net
def _test_overlapping_names(
self,
inputs0: List[Text] = ['i0', 'i1'],
inputs1: List[Text] = ['i2', 'i3'],
outputs0: List[Text] = ['o0', 'o1'],
outputs1: List[Text] = ['o2', 'o3'],
value_info0: List[Text] = ['v0', 'v1'],
value_info1: List[Text] = ['v2', 'v3'],
initializer0: List[Text] = ['init0', 'init1'],
initializer1: List[Text] = ['init2', 'init3'],
sparse_initializer0: List[Text] = ['sparse_init0', 'sparse_init1'],
sparse_initializer1: List[Text] = ['sparse_init2', 'sparse_init3'],
) -> None:
n0 = [
helper.make_node('Identity',
inputs=[inputs0[i]],
outputs=[outputs0[i]])
for i in range(len(inputs0))
]
i0 = [
helper.make_tensor_value_info(inputs0[i], TensorProto.FLOAT, [])
for i in range(len(inputs0))
]
o0 = [
helper.make_tensor_value_info(outputs0[i], TensorProto.FLOAT, [])
for i in range(len(outputs0))
]
vi0 = [
helper.make_tensor_value_info(value_info0[i], TensorProto.FLOAT,
[]) for i in range(len(value_info0))
]
init0 = [
helper.make_tensor(name=initializer0[i],
data_type=TensorProto.INT64,
dims=(),
vals=[1]) for i in range(len(initializer0))
]
sparse_init0 = [
_make_sparse_tensor(sparse_initializer0[i])
for i in range(len(sparse_initializer0))
]
n1 = [
helper.make_node('Identity',
inputs=[inputs1[i]],
outputs=[outputs1[i]])
for i in range(len(inputs1))
]
i1 = [
helper.make_tensor_value_info(inputs1[i], TensorProto.FLOAT, [])
for i in range(len(inputs1))
]
o1 = [
helper.make_tensor_value_info(outputs1[i], TensorProto.FLOAT, [])
for i in range(len(outputs1))
]
vi1 = [
helper.make_tensor_value_info(value_info1[i], TensorProto.FLOAT,
[]) for i in range(len(value_info1))
]
init1 = [
helper.make_tensor(name=initializer1[i],
data_type=TensorProto.INT64,
dims=(),
vals=[1]) for i in range(len(initializer1))
]
sparse_init1 = [
_make_sparse_tensor(sparse_initializer1[i])
for i in range(len(sparse_initializer1))
]
ops = [helper.make_opsetid("", 10)]
m0 = helper.make_model(helper.make_graph(
nodes=n0,
name='g0',
inputs=i0,
outputs=o0,
value_info=vi0,
initializer=init0,
sparse_initializer=sparse_init0),
producer_name='test',
opset_imports=ops)
m1 = helper.make_model(helper.make_graph(
nodes=n1,
name='g1',
inputs=i1,
outputs=o1,
value_info=vi1,
initializer=init1,
sparse_initializer=sparse_init1),
producer_name='test',
opset_imports=ops)
overlap = compose.check_overlapping_names(m0.graph, m1.graph)
i = 0
overlapping_inputs = list(set(inputs0) & set(inputs1))
overlapping_outputs = list(set(outputs0) & set(outputs1))
overlapping_edges = list(set(overlapping_inputs + overlapping_outputs))
if len(overlapping_edges) > 0:
self.assertEqual(overlap[i], ('edge', overlapping_edges))
i += 1
overlapping_vis = list(set(value_info0) & set(value_info1))
if len(overlapping_vis) > 0:
self.assertEqual(overlap[i], ('value_info', overlapping_vis))
i += 1
overlapping_init = list(set(initializer0) & set(initializer1))
if len(overlapping_init) > 0:
self.assertEqual(overlap[i], ('initializer', overlapping_init))
i += 1
overlapping_sparse_init = list(
set(sparse_initializer0) & set(sparse_initializer1))
if len(overlapping_sparse_init) > 0:
expected_overlap = []
for overlapping_name in overlapping_sparse_init:
expected_overlap.append(overlapping_name + '_values')
expected_overlap.append(overlapping_name + '_idx')
self.assertEqual(overlap[i],
('sparse_initializer', expected_overlap))
i += 1
m0_new = compose.add_prefix(m0, prefix='g0/')
overlap = compose.check_overlapping_names(m0_new.graph, m1.graph)
self.assertEqual(0, len(overlap))
def caffe2_net_to_onnx_graph(cls,
predict_net,
init_net=None,
value_info=None):
if value_info is None:
value_info = {}
if not isinstance(value_info, dict):
raise ValueError('Please pass value_info as a '
'name -> (type, shape) dictionary')
cls._ssa_rewrite(predict_net, init_net, value_info)
if init_net:
initializer = cls.caffe2_init_net_to_initializer(init_net)
value_info.update({
init.name: (init.data_type, init.dims)
for init in initializer
})
else:
initializer = []
# Check whether we have got type shape info of all input
missing = (set(list(predict_net.external_input)) -
set(value_info.keys()))
if missing:
raise RuntimeError(
'Could not find value info of inputs: {}'.format(
', '.join(missing)))
inputs = {}
for name in predict_net.external_input:
elem_type, shape = value_info[name]
inputs[name] = np.random.randn(*shape).astype(
mapping.TENSOR_TYPE_TO_NP_TYPE[elem_type])
ws, outputs = c2_native_run_net(init_net, predict_net, inputs)
for name in predict_net.external_output:
output = outputs[name]
elem_type = mapping.NP_TYPE_TO_TENSOR_TYPE[output.dtype]
shape = output.shape
value_info[name] = (elem_type, shape)
graph_def = GraphProto()
graph_def.name = predict_net.name
graph_def.initializer.extend(initializer)
# This is a mapping from Caffe2 names to ONNX names
graph_def.input.extend(
make_tensor_value_info(name=name,
elem_type=value_info[name][0],
shape=value_info[name][1])
for name in predict_net.external_input)
dummy_name(
cls._all_names_in_net(predict_net)
| cls._all_names_in_net(init_net))
for op in predict_net.op:
shapes = {}
for name in itertools.chain(op.input, op.output):
blob = ws.FetchBlob(name)
if hasattr(blob, 'shape'):
shapes[name] = blob.shape
graph_def.node.extend(cls.caffe2_op_to_onnx_node(op,
shapes=shapes))
all_output = set(
sum((list(node.output) for node in graph_def.node),
[init.name for init in graph_def.initializer]))
redundant_output = set(vi.name for vi in graph_def.output) - all_output
if redundant_output:
logger.warning(
'There are graph output not produced by any node or initializer: {}'
'! Will drop them.'.format(', '.join(redundant_output)))
graph_def.output.extend(
make_tensor_value_info(name=name,
elem_type=value_info[name][0],
shape=value_info[name][1])
for name in predict_net.external_output if name in all_output)
cls._annotate_consumed(graph_def)
checker.check_graph(graph_def)
return graph_def
def quantitize_graph(g, verbose=False):
"""Quantitize graph."""
new_weights = []
quantitized_weights = []
nodes = []
remap = {}
remove = []
for i, w in enumerate(g.initializer):
# only quantitize float32
if w.data_type != onnx_pb.TensorProto.FLOAT:
continue
w_np = numpy_helper.to_array(w)
# only look at sizes >= 32 elements
if w_np.size < 32:
continue
# weights we want to quantitize
remove.append(i)
name = w.name
if verbose:
log.info("quantitizing %s", name)
w_quant, zp, scale = eight_bit_quantitize(w_np)
nw = numpy_helper.from_array(w_quant, name=name)
if verbose:
w_dequant = eight_bit_dequantitize(w_quant, zp, scale)
rtol = np.abs(w_dequant - w_np)
s = {}
for j in [1.0, 5.0, 10.0, 20.0]:
above_rtol = np.sum(rtol > np.abs(j * w_np / 100.)) / w_np.size
s["> " + str(j) + "%"] = "{:.2f}".format(100. * above_rtol)
log.info("above_rtol: %s", str(s))
log.info("raw: %s", stats(w_np))
log.info("quant: %s", stats(w_dequant))
output_name = _compose_quantitize(nodes, new_weights, zp, scale, name)
remap[name] = output_name
quantitized_weights.append(nw)
# few things to do to initializers and graph inputs:
# 1. remove initializers that got quantitized
for i in reversed(remove):
del g.initializer[i]
# 2. add quantitized to initializers
g.initializer.extend(new_weights)
g.initializer.extend(quantitized_weights)
# 3. modify the type of weights that we quantitized
modified = {w.name: w for w in quantitized_weights}
new_inputs = []
remove = []
for i, inp in enumerate(g.input):
w = modified.get(inp.name)
if w is not None:
new_inputs.append(
helper.make_tensor_value_info(w.name, w.data_type, w.dims))
remove.append(i)
for i in reversed(remove):
del g.input[i]
# 4. add new weights as inputs
for w in new_weights:
tv = helper.make_tensor_value_info(w.name, w.data_type, w.dims)
new_inputs.append(tv)
g.input.extend(new_inputs)
# 5. rewrite consumers of the quantitized weights
for node in g.node:
for i, name in enumerate(node.input):
new_name = remap.get(name)
if new_name is not None:
node.input[i] = new_name
# 6. add composed nodes to graph, new nodes in the front
nodes.extend(g.node)
del g.node[:]
g.node.extend(nodes)
return g
def test_overlapping_function_names(self) -> None:
'''
Tests error checking when the name of local function entries overlaps
'''
ops = [helper.make_opsetid("", 10), helper.make_opsetid("local", 10)]
def _make_function(
domain: Text,
fname: Text,
inputs: List[Text],
outputs: List[Text],
nodes: List[NodeProto],
) -> FunctionProto:
f = FunctionProto()
f.domain = domain
f.name = fname
f.input.extend(inputs)
f.output.extend(outputs)
f.node.extend(nodes)
f.opset_import.extend(ops)
return f
ops = [helper.make_opsetid("", 10), helper.make_opsetid("local", 10)]
g = GraphProto()
g.input.extend([
helper.make_tensor_value_info('x0', TensorProto.FLOAT, []),
helper.make_tensor_value_info('x1', TensorProto.FLOAT, [])
])
g.output.extend([
helper.make_tensor_value_info('y', TensorProto.FLOAT, []),
])
g.node.extend([
helper.make_node('f1',
domain='local',
inputs=['x0', 'x1'],
outputs=['y'])
])
g1 = GraphProto()
g1.CopyFrom(g)
g1.name = 'g1'
m1 = helper.make_model(g1, producer_name='test', opset_imports=ops)
m1.functions.extend([
_make_function(
'local', 'f1', ['x0', 'x1'], ['y'],
[helper.make_node('Add', inputs=['x0', 'x1'], outputs=['y'])])
])
checker.check_model(m1)
g2 = GraphProto()
g2.CopyFrom(g)
g2.name = 'g2'
m2 = helper.make_model(g2, producer_name='test', opset_imports=ops)
m2.functions.extend([
_make_function(
'local', 'f1', ['x0', 'x1'], ['y'],
[helper.make_node('Mul', inputs=['x0', 'x1'], outputs=['y'])])
])
checker.check_model(m2)
m = compose.merge_models(m1,
m2,
io_map=[('y', 'x0'), ('y', 'x1')],
prefix1='m1/',
prefix2='m2/')
checker.check_model(m)
nodes = [n.op_type for n in m.graph.node]
self.assertEqual(['m1/f1', 'm2/f1'], nodes)
functions = [f.name for f in m.functions]
self.assertEqual(['m1/f1', 'm2/f1'], functions)
g3 = GraphProto()
g3.CopyFrom(g)
g3.name = 'g3'
g3.node[0].op_type = 'f2'
m3 = helper.make_model(g3, producer_name='test', opset_imports=ops)
m3.functions.extend([
_make_function('local', 'f1', ['x0', 'x1'], ['y'], [
helper.make_node('Add', inputs=['x0', 'x1'], outputs=['y0']),
helper.make_node('Mul', inputs=['x0', 'x1'], outputs=['y1']),
helper.make_node('Add', inputs=['y0', 'y1'], outputs=['y'])
]),
_make_function('local', 'f2', ['x0', 'x1'], ['y'], [
helper.make_node(
'f1', domain='local', inputs=['x0', 'x1'], outputs=['y0']),
helper.make_node('Mul', inputs=['x0', 'x1'], outputs=['y1']),
helper.make_node('Add', inputs=['y0', 'y1'], outputs=['y'])
])
])
checker.check_model(m3)
m = compose.merge_models(m1,
m3,
io_map=[('y', 'x0'), ('y', 'x1')],
prefix1='m1/',
prefix2='m3/')
checker.check_model(m)
nodes = [n.op_type for n in m.graph.node]
self.assertEqual(['m1/f1', 'm3/f2'], nodes)
functions = [f.name for f in m.functions]
self.assertEqual(['m1/f1', 'm3/f1', 'm3/f2'], functions)
self.assertEqual(['Add'], [n.op_type for n in m.functions[0].node])
self.assertEqual(['Add', 'Mul', 'Add'],
[n.op_type for n in m.functions[1].node])
self.assertEqual(['m3/f1', 'Mul', 'Add'],
[n.op_type for n in m.functions[2].node])
def GenerateModel(model_name):
nodes = [
helper.make_node("Gather", ["embed_weights", "input_1"],
["gather_out"], "gather"),
helper.make_node("Add", ["gather_out", "add_q_weight"], ["add_q_out"],
"add_q"),
helper.make_node("Add", ["gather_out", "add_k_weight"], ["add_k_out"],
"add_k"),
helper.make_node("Add", ["gather_out", "add_v_weight"], ["add_v_out"],
"add_v"),
helper.make_node(
"Concat",
["add_q_out", "add_k_out", "add_v_out"],
["concat_out"],
"concat",
axis=0,
),
helper.make_node("Add", ["add_qkv_weight", "concat_out"], ["add_out"],
"add"),
helper.make_node(
"ReduceSum",
["add_out"],
["predictions"],
"reduce_sum_1",
axes=[0],
keepdims=1,
),
]
embed_weights = np.random.uniform(-1, 1, 8000).tolist()
add_q_weight = [
-0.23681640625,
-0.16552734375,
0.2191162109375,
-0.1756591796875,
-0.03460693359375,
-0.05316162109375,
-0.336181640625,
-0.253662109375,
]
add_k_weight = [
0.0246734619140625,
0.011993408203125,
0.0178375244140625,
0.00998687744140625,
0.0255126953125,
0.076416015625,
-0.040771484375,
0.0107879638671875,
]
add_v_weight = [
-0.005893707275390625,
-0.00916290283203125,
0.04541015625,
0.0159454345703125,
-0.0029163360595703125,
-0.03472900390625,
0.0535888671875,
0.0091094970703125,
]
initializers = [ # initializers
helper.make_tensor("embed_weights", TensorProto.FLOAT, [1000, 8],
embed_weights),
helper.make_tensor("add_q_weight", TensorProto.FLOAT, [8],
add_q_weight),
helper.make_tensor("add_k_weight", TensorProto.FLOAT, [8],
add_k_weight),
helper.make_tensor("add_v_weight", TensorProto.FLOAT, [8],
add_v_weight),
helper.make_tensor("add_qkv_weight", TensorProto.FLOAT, [1], [1.0]),
]
graph = helper.make_graph(
nodes,
"ConcatThreeInputs", # name
[
helper.make_tensor_value_info("input_1", TensorProto.INT64,
["batch", "seq_len"])
], # inputs
[ # outputs
helper.make_tensor_value_info("predictions", TensorProto.FLOAT,
[1, 1, 8]),
],
initializers,
)
model = helper.make_model(graph)
onnx.save(model, model_name)
