def test_attr_doc_string(self): # type: () -> None
attr = helper.make_attribute("a", "value")
self.assertEqual(attr.name, "a")
self.assertEqual(attr.doc_string, "")
attr = helper.make_attribute("a", "value", "doc")
self.assertEqual(attr.name, "a")
self.assertEqual(attr.doc_string, "doc")
def test_attr_float(self): # type: () -> None
# float
attr = helper.make_attribute("float", 1.)
self.assertEqual(attr.name, "float")
self.assertEqual(attr.f, 1.)
checker.check_attribute(attr)
# float with scientific
attr = helper.make_attribute("float", 1e10)
self.assertEqual(attr.name, "float")
self.assertEqual(attr.f, 1e10)
checker.check_attribute(attr)
def _common_caffe2_arg_to_onnx_attr(cls, op_def, arg):
# name
op_type = op_def.type
if op_type in cls._per_op_renamed_args:
name = cls._per_op_renamed_args[op_type].get(
arg.name, arg.name)
else:
name = cls._global_renamed_args.get(arg.name, arg.name)
# value
if arg.HasField('f'):
value = arg.f
elif arg.HasField('i'):
value = arg.i
elif arg.HasField('s'):
value = arg.s
elif arg.floats:
value = arg.floats
elif arg.ints:
value = arg.ints
elif arg.strings:
value = arg.strings
else:
raise ValueError('Could not find data field in arg: {}'.format(arg))
if name in cls._blacklist_caffe2_args:
assert value in cls._blacklist_caffe2_args[arg.name]
return None
return helper.make_attribute(name, value)
def test_attr_string(self): # type: () -> None
# bytes
attr = helper.make_attribute("str", b"test")
self.assertEqual(attr.name, "str")
self.assertEqual(attr.s, b"test")
checker.check_attribute(attr)
# unspecified
attr = helper.make_attribute("str", "test")
self.assertEqual(attr.name, "str")
self.assertEqual(attr.s, b"test")
checker.check_attribute(attr)
# unicode
attr = helper.make_attribute("str", u"test")
self.assertEqual(attr.name, "str")
self.assertEqual(attr.s, b"test")
checker.check_attribute(attr)
def apply_trans(k, dim=2, ks=None):
ks = ks or (k + 's')
if dim == 2:
k_h, k_w = k + '_h', k + '_w'
else:
k_t, k_l, k_b, k_r = k + '_t', k + '_l', k + '_b', k + '_r'
vals = None
if (dim == 2 and k_h in attrs and k_w in attrs):
vals = [attrs[k_h].i, attrs[k_w].i]
del attrs[k_h]
del attrs[k_w]
elif (dim == 4 and
k_t in attrs and k_l in attrs and k_b in attrs and k_r in attrs):
vals = [attrs[k_t].i,
attrs[k_l].i,
attrs[k_b].i,
attrs[k_r].i]
del attrs[k_t]
del attrs[k_l]
del attrs[k_b]
del attrs[k_r]
elif k in attrs:
vals = [attrs[k].i] * dim
del attrs[k]
if vals and not node.op_type.startswith('Global'):
attrs[ks] = helper.make_attribute(ks, vals)
def _create_concat(cls, op_def, shapes):
node = cls._common_caffe2_op_to_onnx_node(op_def, shapes)
if len(node.output) == 2:
del node.output[1]
explicit_axis = any(arg.name == 'axis' for arg in op_def.arg)
if not explicit_axis:
node.attribute.extend([helper.make_attribute('axis', 1)])
return node
def test_attr_repeated_graph_proto(self): # type: () -> None
graphs = [GraphProto(), GraphProto()]
graphs[0].name = "a"
graphs[1].name = "b"
attr = helper.make_attribute("graphs", graphs)
self.assertEqual(attr.name, "graphs")
self.assertEqual(list(attr.graphs), graphs)
checker.check_attribute(attr)
def test_attr_repeated_graph_proto(self): # type: () -> None
graphs = [GraphProto(), GraphProto()]
graphs[0].name = "a"
graphs[1].name = "b"
attr = helper.make_attribute("graphs", graphs)
self.assertEqual(attr.name, "graphs")
self.assertEqual(list(attr.graphs), graphs)
checker.check_attribute(attr)
def _create_concat(cls, op_def, shapes):
node = cls._common_caffe2_op_to_onnx_node(op_def, shapes)
if len(node.output) == 2:
del node.output[1]
explicit_axis = any(arg.name == 'axis' for arg in op_def.arg)
if not explicit_axis:
node.attribute.extend([helper.make_attribute('axis', 1)])
return node
def ImageDataONNXExporter(op_def, shape_dict, ws):
node_proto, const_tensors = CommonONNXExporter(op_def, shape_dict)
node_proto.op_type = 'ATen' # Template
node_proto.attribute.extend([make_attribute('op_type', 'ImageData')])
for arg in op_def.arg:
if arg.name == 'mean_values':
node_proto.attribute.extend(
[make_attribute('mean_values', arg.floats)])
elif arg.name == 'std_values':
node_proto.attribute.extend(
[make_attribute('std_values', arg.floats)])
elif arg.name == 'dtype':
node_proto.attribute.extend([make_attribute('dtype', arg.s)])
elif arg.name == 'data_format':
node_proto.attribute.extend([make_attribute('data_format', arg.s)])
return node_proto, const_tensors
def fuse(self, node, input_name_to_nodes, output_name_to_node):
add = self.model.get_parent(node, 0, output_name_to_node)
# In some models there is input_ids->gather->add->LayerNorm and one of input of the
# add node is initializer with fixed shape which should not be fused into SkipLayerNorm
if add is None:
return
for add_input in add.input:
if self.model.get_initializer(add_input) != None:
return
# The number of input node of add should be 2
if len(self.model.get_parents(add)) != 2:
return
if self.shape_infer_helper is not None:
if not self.shape_infer_helper.compare_shape(
add.input[0], add.input[1]):
return
else:
logger.warning(
"symbolic shape infer failed. it's safe to ignore this message if there is no issue with optimized model"
)
gather_path = self.model.match_parent_path(add, ['Gather'], [None])
if gather_path is not None and self.model.find_graph_input(
gather_path[0].input[1]) is None:
if self.model.match_parent_path(gather_path[0],
['ConstantOfShape'], [1]) is None:
return
if add is not None and add.op_type == 'Add' and self.model.is_safe_to_fuse_nodes(
[add, node], node.output, input_name_to_nodes,
output_name_to_node):
self.nodes_to_remove.extend([add, node])
inputs = [add.input[0], add.input[1], node.input[1], node.input[2]]
normalize_node = helper.make_node("SkipLayerNormalization",
inputs=inputs,
outputs=[node.output[0]],
name=self.model.create_node_name(
"SkipLayerNormalization",
name_prefix="SkipLayerNorm"))
normalize_node.domain = "com.microsoft"
# Pass attribute "epsilon" from layernorm node to SkipLayerNormalization
for att in node.attribute:
if att.name == 'epsilon':
normalize_node.attribute.extend([att])
# Set default epsilon if no epsilon exists from layernorm
if len(normalize_node.attribute) == 0:
normalize_node.attribute.extend(
[helper.make_attribute("epsilon", 1.0E-12)])
self.nodes_to_add.append(normalize_node)
def ROIAlignONNXExporter(op_def, shape_dict, ws):
node_proto, const_tensors = CommonONNXExporter(op_def, shape_dict)
node_proto.op_type = 'ATen' # Template
node_proto.attribute.extend([make_attribute('op_type', 'ROIAlign')])
for arg in op_def.arg:
if arg.name == 'pool_h':
node_proto.attribute.extend([make_attribute('pool_h', arg.i)])
elif arg.name == 'pool_w':
node_proto.attribute.extend([make_attribute('pool_w', arg.i)])
elif arg.name == 'spatial_scale':
node_proto.attribute.extend(
[make_attribute('spatial_scale', arg.f)])
elif arg.name == 'sampling_ratio':
node_proto.attribute.extend(
[make_attribute('sampling_ratio', arg.i)])
return node_proto, const_tensors
def fuse_attention_node(
self,
matmul_before_split,
add_before_split,
past,
present,
input,
reshape_qkv,
mask,
):
attention_node_name = self.model.create_node_name("GptAttention")
int32_mask = self.cast_attention_mask(mask)
output = reshape_qkv.output[0]
i = 1 if (add_before_split.input[0] == matmul_before_split.output[0]) else 0
attention_node = helper.make_node(
"Attention",
inputs=[
input,
matmul_before_split.input[1],
add_before_split.input[i],
int32_mask,
past,
],
outputs=[output, present],
name=attention_node_name,
)
attention_node.domain = "com.microsoft"
attention_node.attribute.extend(
[
helper.make_attribute("num_heads", self.num_heads),
helper.make_attribute("unidirectional", 0), # unidirectional shall not be ON for 4D attention mask
]
)
nodes_to_add = [attention_node]
self.nodes_to_add.extend(nodes_to_add)
for node in nodes_to_add:
self.node_name_to_graph_name[node.name] = self.this_graph_name
self.nodes_to_remove.append(reshape_qkv)
# we rely on prune_graph() to clean old subgraph nodes
self.prune_graph = True
def create_attention_node(
self,
fc_weight,
fc_bias,
gemm_qkv,
past,
present,
input,
output,
mask,
is_unidirectional,
):
attention_node_name = self.model.create_node_name("GptAttention")
attention_node = helper.make_node(
"Attention",
inputs=[input, fc_weight, fc_bias, mask, past],
outputs=[attention_node_name + "_output", present],
name=attention_node_name,
)
attention_node.domain = "com.microsoft"
attention_node.attribute.extend([
helper.make_attribute("num_heads", self.num_heads),
helper.make_attribute("unidirectional",
1 if is_unidirectional else 0),
])
matmul_node = helper.make_node(
"MatMul",
inputs=[attention_node_name + "_output", gemm_qkv.input[1]],
outputs=[attention_node_name + "_matmul_output"],
name=attention_node_name + "_matmul",
)
add_node = helper.make_node(
"Add",
inputs=[attention_node_name + "_matmul_output", gemm_qkv.input[2]],
outputs=[output],
name=attention_node_name + "_add",
)
self.nodes_to_add.extend([attention_node, matmul_node, add_node])
self.node_name_to_graph_name[
attention_node.name] = self.this_graph_name
self.node_name_to_graph_name[matmul_node.name] = self.this_graph_name
self.node_name_to_graph_name[add_node.name] = self.this_graph_name
def create_attention_node(self, mask_index: str, matmul: NodeProto, add: NodeProto, num_heads: int,
hidden_size: int, input: str, output: str, add_qk_str: str) -> Union[NodeProto, None]:
assert num_heads > 0
if hidden_size > 0 and (hidden_size % num_heads) != 0:
logger.debug(f"input hidden size {hidden_size} is not a multiple of num of heads {num_heads}")
return None
weight = self.model.get_initializer(matmul.input[1])
bias = self.model.get_initializer(add.input[1]) or self.model.get_initializer(add.input[0])
if weight is None or bias is None:
return None
qkv_weight = NumpyHelper.to_array(weight)
qkv_bias = NumpyHelper.to_array(bias)
attention_node_name = self.model.create_node_name('Attention')
weight = helper.make_tensor(name=attention_node_name + '_qkv_weight',
data_type=TensorProto.FLOAT,
dims=[hidden_size, 3 * hidden_size],
vals=qkv_weight.flatten().tolist())
# Sometimes weights and bias are stored in fp16
if weight.data_type == 10:
weight.CopyFrom(numpy_helper.from_array(NumpyHelper.to_array(weight).astype(np.float16), weight.name))
self.model.add_initializer(weight, self.this_graph_name)
bias = helper.make_tensor(name=attention_node_name + '_qkv_bias',
data_type=TensorProto.FLOAT,
dims=[3 * hidden_size],
vals=qkv_bias.flatten().tolist())
if bias.data_type == 10:
bias.CopyFrom(numpy_helper.from_array(NumpyHelper.to_array(bias).astype(np.float16), bias.name))
self.model.add_initializer(bias, self.this_graph_name)
attention_inputs = [input, attention_node_name + '_qkv_weight', attention_node_name + '_qkv_bias']
if mask_index is not None:
attention_inputs.append(mask_index)
else:
attention_inputs.append("")
if add_qk_str is not None:
attention_inputs.append("")
attention_inputs.append(add_qk_str)
attention_node = helper.make_node('Attention',
inputs=attention_inputs,
outputs=[output],
name=attention_node_name)
attention_node.domain = "com.microsoft"
attention_node.attribute.extend([helper.make_attribute("num_heads", num_heads)])
return attention_node
def _create_gemm(cls, op, op_t):
"""
get a onnx node from singa gemm operator
Args:
op: a given operator
Args:
op_t: the tensor of the operator
Returns:
the onnx node
"""
node = cls._common_singa_tensor_to_onnx_node(op, op_t)
node.attribute.extend([
helper.make_attribute('alpha', float(op.alpha)),
helper.make_attribute('beta', float(op.beta)),
helper.make_attribute('transA', 1 if op.transA else 0),
helper.make_attribute('transB', 1 if op.transB else 0),
])
return node
def test_node_with_arg(self): # type: () -> None
self.assertTrue(defs.has("Relu"))
# Note: Relu actually does not need an arg, but let's
# test it.
node_def = helper.make_node("Relu", ["X"], ["Y"], arg_value=1)
self.assertEqual(node_def.op_type, "Relu")
self.assertEqual(list(node_def.input), ["X"])
self.assertEqual(list(node_def.output), ["Y"])
self.assertEqual(len(node_def.attribute), 1)
self.assertEqual(node_def.attribute[0],
helper.make_attribute("arg_value", 1))
def _create_batch_norm(cls, op, op_t):
"""
get a onnx node from singa _BatchNorm2d operator
Args:
op: a given operator
Args:
op_t: the tensor of the operator
Returns:
the onnx node
"""
nodes = []
# firstly we add the running mean and var nodes
running_values = [op.running_mean, op.running_var]
for srcop, _, y, _ in op.src:
if y is None and srcop.name in cls._unhandled_operators:
node = cls._common_singa_tensor_to_onnx_node(srcop, op_t)
running_value = running_values.pop(0)
vals = tensor.to_numpy(
tensor.from_raw_tensor(running_value)).astype(float)
node.attribute.extend([
helper.make_attribute(
'value',
helper.make_tensor(
name=srcop.name,
data_type=TensorProto.FLOAT,
dims=[len(vals)],
vals=vals,
))
])
nodes.append(node)
# then we add the batchnorm op itself
epsilon = 1e-5 # the epsilon value used in singa
node = cls._common_singa_tensor_to_onnx_node(op, op_t)
node.attribute.extend([
helper.make_attribute('momentum', op.handle.factor),
helper.make_attribute('epsilon', epsilon),
])
nodes.append(node)
return nodes
def test_attr_int(self): # type: () -> None
# integer
attr = helper.make_attribute("int", 3)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 3)
checker.check_attribute(attr)
# long integer
attr = helper.make_attribute("int", 5)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 5)
checker.check_attribute(attr)
# octinteger
attr = helper.make_attribute("int", 0o1701)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 0o1701)
checker.check_attribute(attr)
# hexinteger
attr = helper.make_attribute("int", 0x1701)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 0x1701)
checker.check_attribute(attr)
def clean_graph(self):
output_name_to_node = self.output_name_to_node()
nodes_to_remove = []
for node in self.nodes():
# Before:
# input_ids --> Shape --> Gather(indices=0) --> Unsqueeze ------+
# | |
# | v
# +----> Shape --> Gather(indices=1) --> Unsqueeze---> Concat --> ConstantOfShape -->Cast --> EmbedLayerNormaliation/ReduceSum
# After:
# input_ids --> Shape --> ConstantOfShape -->Cast --> EmbedLayerNormaliation/ReduceSum
# TODO: merge ConstantOfShape -->Cast to ConstantOfShape (need update the data type of value)
op_input_id = {
"EmbedLayerNormalization": 1,
"ReduceSum": 0,
"Attention": 3
}
if node.op_type in op_input_id:
i = op_input_id[node.op_type]
parent_nodes = self.match_parent_path(node, [
'Cast', 'ConstantOfShape', 'Concat', 'Unsqueeze', 'Gather',
'Shape'
], [i, 0, 0, 0, 0, 0], output_name_to_node)
if parent_nodes is not None:
cast, constantOfShape, concat, unsqueeze, gather, shape = parent_nodes
if shape.input[0] == self.graph().input[0].name:
constantOfShape.input[0] = shape.output[0]
output_name_to_node = self.output_name_to_node()
if node.op_type == 'Attention':
# Before:
# input_ids --> Shape -->ConstantOfShape -->Cast --> ReduceSum --> Attention
# After:
# remove this path, and remove the optional mask_index input of Attention node.
parent_nodes = self.match_parent_path(
node, ['ReduceSum', 'Cast', 'ConstantOfShape', 'Shape'],
[3, 0, 0, 0], output_name_to_node)
if parent_nodes is not None:
if parent_nodes[-1].input[0] == self.graph().input[0].name:
attention_node = helper.make_node(
'Attention',
inputs=node.input[0:len(node.input) - 1],
outputs=node.output,
name=node.name + "_remove_mask")
attention_node.domain = "com.microsoft"
attention_node.attribute.extend([
helper.make_attribute("num_heads", self.num_heads)
])
self.add_node(
attention_node,
self.get_graph_by_node(attention_node).name)
nodes_to_remove.append(node)
self.remove_nodes(nodes_to_remove)
def test_attr_int(self): # type: () -> None
# integer
attr = helper.make_attribute("int", 3)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 3)
checker.check_attribute(attr)
# long integer
attr = helper.make_attribute("int", 5)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 5)
checker.check_attribute(attr)
# octinteger
attr = helper.make_attribute("int", 0o1701)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 0o1701)
checker.check_attribute(attr)
# hexinteger
attr = helper.make_attribute("int", 0x1701)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 0x1701)
checker.check_attribute(attr)
def test_attr_int(self):
# integer
attr = helper.make_attribute("int", 3)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 3)
self.assertTrue(helper.is_attribute_legal(attr))
# long integer
attr = helper.make_attribute("int", 5)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 5)
self.assertTrue(helper.is_attribute_legal(attr))
# octinteger
attr = helper.make_attribute("int", 0o1701)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 0o1701)
self.assertTrue(helper.is_attribute_legal(attr))
# hexinteger
attr = helper.make_attribute("int", 0x1701)
self.assertEqual(attr.name, "int")
self.assertEqual(attr.i, 0x1701)
self.assertTrue(helper.is_attribute_legal(attr))
def test_attr_string(self): # type: () -> None
# bytes
attr = helper.make_attribute("str", b"test")
self.assertEqual(attr.name, "str")
self.assertEqual(attr.s, b"test")
checker.check_attribute(attr)
# unspecified
attr = helper.make_attribute("str", "test")
self.assertEqual(attr.name, "str")
self.assertEqual(attr.s, b"test")
checker.check_attribute(attr)
# unicode
attr = helper.make_attribute("str", u"test")
self.assertEqual(attr.name, "str")
self.assertEqual(attr.s, b"test")
checker.check_attribute(attr)
# empty str
attr = helper.make_attribute("str", "")
self.assertEqual(attr.name, "str")
self.assertEqual(helper.get_attribute_value(attr), b"")
checker.check_attribute(attr)
def create_attention_node(self, gemm, gemm_qkv, input, output):
attention_node_name = self.model.create_node_name('Attention')
attention_node = helper.make_node('Attention',
inputs=[input, gemm.input[1], gemm.input[2]],
outputs=[attention_node_name + "_output"],
name=attention_node_name)
attention_node.domain = "com.microsoft"
attention_node.attribute.extend(
[helper.make_attribute("num_heads", self.num_heads),
helper.make_attribute("unidirectional", 1)])
matmul_node = helper.make_node('MatMul',
inputs=[attention_node_name + "_output", gemm_qkv.input[1]],
outputs=[attention_node_name + "_matmul_output"],
name=attention_node_name + "_matmul")
add_node = helper.make_node('Add',
inputs=[attention_node_name + "_matmul_output", gemm_qkv.input[2]],
outputs=[output],
name=attention_node_name + "_add")
self.nodes_to_add.extend([attention_node, matmul_node, add_node])
def process_mask(self, input: str) -> str:
if self.mask_format == AttentionMaskFormat.NoMask:
return None
if input in self.mask_indice:
return self.mask_indice[input]
# Add cast to convert int64 to int32
if self.model.find_graph_input(input):
casted, input_name = self.utils.cast_graph_input_to_int32(input)
else:
input_name, cast_node = self.utils.cast_input_to_int32(input)
casted = True
if casted:
self.mask_casted[input] = input_name
# Attention supports int32 attention mask (2D) since 1.4.0
if self.mask_format == AttentionMaskFormat.AttentionMask:
self.mask_indice[input] = input_name
return input_name
# Add a mask processing node to convert attention mask to mask index (1D)
output_name = self.model.create_node_name("mask_index")
mask_index_node = helper.make_node(
"ReduceSum",
inputs=[input_name],
outputs=[output_name],
name=self.model.create_node_name("ReduceSum", "MaskReduceSum"),
)
mask_index_node.attribute.extend([
helper.make_attribute("axes", [1]),
helper.make_attribute("keepdims", 0)
])
self.model.add_node(mask_index_node)
self.mask_indice[input] = output_name
return output_name
def ArgReduceONNXExporter(op_def, shape_dict, ws):
node_proto, const_tensors = CommonONNXExporter(op_def, shape_dict)
# ONNX requires indices only, remove the values
indices = node_proto.output[0]
node_proto.ClearField('output')
node_proto.output.extend([indices])
for arg in op_def.arg:
if arg.name == 'axis':
node_proto.attribute.extend([make_attribute('axis', arg.i)])
elif arg.name == 'keep_dims':
node_proto.attribute.extend([make_attribute('keepdims', arg.i)])
elif arg.name == 'top_k':
if arg.i != 1:
raise ValueError('ONNX requires top_k == 1.')
elif arg.name == 'operation':
if arg.s == b'ARGMAX':
node_proto.op_type = 'ArgMax'
elif arg.s == b'ARGMIN':
node_proto.op_type = 'ArgMin'
return node_proto, None
def FlattenONNXExporter(op_def, shape_dict, ws):
node_proto, const_tensors = CommonONNXExporter(op_def, shape_dict)
for arg in op_def.arg:
if arg.name == 'axis':
node_proto.attribute.extend([make_attribute('axis', arg.i)])
elif arg.name == 'num_axes':
if arg.i != -1:
raise ValueError('Excepted num_axes == -1, but got {}.'.format(
arg.i))
elif arg.name == 'keep_axes':
raise ValueError('keep_axes should not be set. (Theano Style).')
return node_proto, None
def test_node_with_arg(self): # type: () -> None
self.assertTrue(defs.has("Relu"))
# Note: Relu actually does not need an arg, but let's
# test it.
node_def = helper.make_node(
"Relu", ["X"], ["Y"],
arg_value=1)
self.assertEqual(node_def.op_type, "Relu")
self.assertEqual(list(node_def.input), ["X"])
self.assertEqual(list(node_def.output), ["Y"])
self.assertEqual(len(node_def.attribute), 1)
self.assertEqual(
node_def.attribute[0],
helper.make_attribute("arg_value", 1))
def PoolNdONNXExporter(op_def, shape_dict, ws):
rank = len(shape_dict[op_def.input[0]]) - 2
node_proto, const_tensors = CommonONNXExporter(op_def, shape_dict)
for arg in op_def.arg:
_assert_data_format(arg)
if arg.name == 'kernel_shape':
node_proto.attribute.extend([
make_attribute('kernel_shape',
_normalize_tuple(arg.ints, rank))
])
elif arg.name == 'strides':
node_proto.attribute.extend(
[make_attribute('strides', _normalize_tuple(arg.ints, rank))])
elif arg.name == 'pads':
node_proto.attribute.extend(
[make_attribute('pads', _normalize_pads(arg.ints, rank))])
elif arg.name == 'padding' and arg.s != b'VALID':
node_proto.attribute.extend([make_attribute('auto_pad', arg.s)])
elif arg.name == 'mode':
if arg.s == 'MAX': node_proto.op_type = 'MaxPool'
elif arg.s == 'AVG': node_proto.op_type = 'AveragePool'
return node_proto, const_tensors
def test_attr_repeated_tensor_proto(self): # type: () -> None
tensors = [
helper.make_tensor(name='a',
data_type=TensorProto.FLOAT,
dims=(1, ),
vals=np.ones(1).tolist()),
helper.make_tensor(name='b',
data_type=TensorProto.FLOAT,
dims=(1, ),
vals=np.ones(1).tolist())
]
attr = helper.make_attribute("tensors", tensors)
self.assertEqual(attr.name, "tensors")
self.assertEqual(list(attr.tensors), tensors)
checker.check_attribute(attr)
def build_onnx_op(node):
"""Build onnx op"""
onnx_node = helper.make_node(node.type, node.input, node.output, name=node.name)
# deal with attributes
attr = []
attr_graphs = node.get_body_graphs()
if attr_graphs:
for attr_name, sub_graph in attr_graphs.items():
copied_sub_graph = copy.deepcopy(sub_graph)
graph_proto = copied_sub_graph.make_graph("graph for " + node.name + " " + attr_name)
attr.append(helper.make_attribute(attr_name, graph_proto))
attr.extend([a for a in node.attr_onnx.values()])
if attr:
onnx_node.attribute.extend(attr)
return onnx_node
def attrs_onnx(self):
"""Return onnx valid attributes"""
schema = get_schema(self.op_type, self.graph.opset, self.domain)
if schema is None and not (self.is_const() or self.is_graph_input()):
logger.debug(
"Node %s uses non-stardard onnx op <%s, %s>, skip attribute check",
self.name,
self.domain,
self.op_type,
)
onnx_attrs = {}
for name, attr in self.attrs.items():
if schema is None or schema.has_attribute(name):
onnx_attrs[name] = helper.make_attribute(name, attr)
return onnx_attrs
def _annotate_consumed(cls, graph_def):
for node in graph_def.node:
schema = defs.get_schema(node.op_type)
consumes = []
for i, _input_name in enumerate(node.input):
consume_type, output_idx = schema.consumed(i)
if consume_type == defs.OpSchema.UseType.CONSUME_ENFORCED:
consumes.append(1)
else:
consumes.append(0)
if any(consumes):
node.attribute.extend([helper.make_attribute(
'consumed_inputs',
consumes,
)])
def _create_flatten(cls, op, op_t):
"""
get a onnx node from singa flatten operator
Args:
op: a given operator
Args:
op_t: the tensor of the operator
Returns:
the onnx node
"""
node = cls._common_singa_tensor_to_onnx_node(op, op_t)
node.attribute.extend([
helper.make_attribute('axis', op.start_axis),
])
return node
def cast_input_to_int32(self, input_name: str):
cast_output = input_name + '_int32'
# Avoid consequent Cast nodes.
inputs = [input_name]
output_name_to_node = self.model.output_name_to_node()
if input_name in output_name_to_node:
parent_node = output_name_to_node[input_name]
if parent_node and parent_node.op_type == 'Cast':
inputs = [parent_node.input[0]]
cast_node = helper.make_node('Cast', inputs=inputs, outputs=[cast_output])
cast_node.attribute.extend([helper.make_attribute("to", int(TensorProto.INT32))])
self.model.add_node(cast_node)
return cast_output, cast_node
def test_attr_sparse_tensor_proto(self) -> None:
dense_shape = [3, 3]
sparse_values = [1.764052391052246, 0.40015721321105957, 0.978738009929657]
values_tensor = helper.make_tensor(name='sparse_values', data_type=TensorProto.FLOAT,
dims=[len(sparse_values)],
vals=np.array(sparse_values).astype(np.float32), raw=False)
linear_indicies = [2, 3, 5]
indicies_tensor = helper.make_tensor(name='indicies', data_type=TensorProto.INT64,
dims=[len(linear_indicies)],
vals=np.array(linear_indicies).astype(np.int64), raw=False)
sparse_tensor = helper.make_sparse_tensor(values_tensor, indicies_tensor, dense_shape)
attr = helper.make_attribute("sparse_attr", sparse_tensor)
self.assertEqual(attr.name, "sparse_attr")
checker.check_sparse_tensor(helper.get_attribute_value(attr))
checker.check_attribute(attr)
def AsTypeONNXExporter(op_def, shape_dict, ws):
node_proto, const_tensors = CommonONNXExporter(op_def, shape_dict)
node_proto.op_type = 'Cast'
if len(node_proto.input) == 0:
raise ValueError('ONNX does not support in-place cast.')
for arg in op_def.arg:
if arg.name == 'dtype':
if arg.s.upper() == b'BOOL':
node_proto.attribute.extend(
[make_attribute('to', TensorProto.BOOL)])
elif arg.s.upper() == b'INT8':
node_proto.attribute.extend(
[make_attribute('to', TensorProto.INT8)])
elif arg.s.upper() == b'UINT8':
node_proto.attribute.extend(
[make_attribute('to', TensorProto.UINT8)])
elif arg.s.upper() == b'INT32':
node_proto.attribute.extend(
[make_attribute('to', TensorProto.INT32)])
elif arg.s.upper() == b'INT64':
node_proto.attribute.extend(
[make_attribute('to', TensorProto.INT64)])
elif arg.s.upper() == b'FLOAT16':
node_proto.attribute.extend(
[make_attribute('to', TensorProto.FLOAT16)])
if arg.s.upper() == b'FLOAT32':
node_proto.attribute.extend(
[make_attribute('to', TensorProto.FLOAT)])
elif arg.s.upper() == b'FLOAT64':
node_proto.attribute.extend(
[make_attribute('to', TensorProto.DOUBLE)])
else:
node_proto.attribute.extend(
[make_attribute('to', TensorProto.UNDEFINED)])
return node_proto, const_tensors
def construct_model_attention_and_matmul(self, output_model_path):
# (input)
# |
# Attention
# |
# MatMul
# |
# (output)
input_name = 'input'
output_name = 'output'
initializers = []
def make_attention_node(input_name, weight_shape, weight_name, bias_shape, bias_name, output_name):
weight_data = np.random.normal(0, 0.1, weight_shape).astype(np.float32)
initializers.append(onnx.numpy_helper.from_array(weight_data, name=weight_name))
bias_data = np.random.normal(0, 0.1, bias_shape).astype(np.float32)
initializers.append(onnx.numpy_helper.from_array(bias_data, name=bias_name))
return onnx.helper.make_node('Attention', [input_name, weight_name, bias_name], [output_name])
def make_matmul_node(input_name, weight_shape, weight_name, output_name):
weight_data = np.random.normal(0, 0.1, weight_shape).astype(np.float32)
initializers.append(onnx.numpy_helper.from_array(weight_data, name=weight_name))
return onnx.helper.make_node('MatMul', [input_name, weight_name], [output_name])
# make attention node
attention_output_name = "attention_output"
attention_node = make_attention_node(input_name, [10, 30], 'qkv.weight', [30], 'qkv.bias', attention_output_name)
attention_node.domain = "com.microsoft"
attention_node.attribute.extend([helper.make_attribute("num_heads", 5)])
# make matmul node
matmul_node = make_matmul_node(attention_output_name, [10, 10], 'matmul.weight', output_name)
# make graph
input_tensor = helper.make_tensor_value_info(input_name, TensorProto.FLOAT, [1, -1, 10])
output_tensor = helper.make_tensor_value_info(output_name, TensorProto.FLOAT, [1, -1, 10])
graph_name = 'attention_test'
graph = helper.make_graph([attention_node, matmul_node], graph_name,
[input_tensor], [output_tensor], initializer=initializers)
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
model.ir_version = onnx.IR_VERSION
onnx.save(model, output_model_path)
def test_attr_repeated_tensor_proto(self): # type: () -> None
tensors = [
helper.make_tensor(
name='a',
data_type=TensorProto.FLOAT,
dims=(1,),
vals=np.ones(1).tolist()
),
helper.make_tensor(
name='b',
data_type=TensorProto.FLOAT,
dims=(1,),
vals=np.ones(1).tolist()
)]
attr = helper.make_attribute("tensors", tensors)
self.assertEqual(attr.name, "tensors")
self.assertEqual(list(attr.tensors), tensors)
checker.check_attribute(attr)
def set_nodeattr(self, name, value):
"""Set a node attribute by name. Data is stored inside the ONNX node's
AttributeProto container. Attribute must be part of get_nodeattr_types."""
try:
(dtype, req, def_val) = self.get_nodeattr_types()[name]
attr = get_by_name(self.onnx_node.attribute, name)
if attr is not None:
# dtype indicates which ONNX Attribute member to use
# (such as i, f, s...)
if dtype == "s":
# encode string attributes
value = value.encode("utf-8")
attr.__setattr__(dtype, value)
else:
# not set, create and insert AttributeProto
attr_proto = helper.make_attribute(name, value)
self.onnx_node.attribute.append(attr_proto)
except KeyError:
raise AttributeError("Op has no such attribute: " + name)
def _create_slice(cls, op_def, shapes):
if len(op_def.input) > 1:
raise Unsupported(
'ONNX Slice operator does not support dynamic slice.')
node = cls._common_caffe2_op_to_onnx_node(op_def, shapes)
attrs = {attr.name: attr for attr in node.attribute}
ndims = len(attrs['starts'].ints)
node.attribute.extend([helper.make_attribute('axes', range(ndims))])
data, = node.input
shape = shapes[data]
ends = attrs['ends'].ints
for i, end in enumerate(ends):
if end >= 0:
continue
if end == -1:
end = shape[i]
else:
end = end + 1
ends[i] = end
return node
def _create_conv_pool_op(cls, op_def, shapes):
node = cls._common_caffe2_op_to_onnx_node(op_def, shapes)
if node.op_type in ['MaxPool', 'AveragePool']:
for i, attr in enumerate(node.attribute):
if attr.name == 'global_pooling' and attr.i:
node.op_type = 'Global{}'.format(node.op_type)
del node.attribute[i]
break
attrs = {attr.name: attr for attr in node.attribute}
def apply_trans(k, dim=2, ks=None):
ks = ks or (k + 's')
if dim == 2:
k_h, k_w = k + '_h', k + '_w'
else:
k_t, k_l, k_b, k_r = k + '_t', k + '_l', k + '_b', k + '_r'
vals = None
if (dim == 2 and k_h in attrs and k_w in attrs):
vals = [attrs[k_h].i, attrs[k_w].i]
del attrs[k_h]
del attrs[k_w]
elif (dim == 4 and
k_t in attrs and k_l in attrs and k_b in attrs and k_r in attrs):
vals = [attrs[k_t].i,
attrs[k_l].i,
attrs[k_b].i,
attrs[k_r].i]
del attrs[k_t]
del attrs[k_l]
del attrs[k_b]
del attrs[k_r]
elif k in attrs:
vals = [attrs[k].i] * dim
del attrs[k]
if vals and not node.op_type.startswith('Global'):
attrs[ks] = helper.make_attribute(ks, vals)
apply_trans('kernel', ks='kernel_shape')
apply_trans('stride')
apply_trans('dilation')
apply_trans('adj')
apply_trans('pad', dim=4)
legacy_pad_attr = attrs.pop('legacy_pad', None)
if legacy_pad_attr:
assert node.op_type.endswith("Pool")
assert not node.op_type.startswith("Global")
input_size = shapes[node.input[0]]
output_size = shapes[node.output[0]]
assert len(output_size) == 4
if legacy_pad_attr.i == caffe2_legacy_pb2.NOTSET:
pass
elif legacy_pad_attr.i == caffe2_legacy_pb2.VALID:
assert not 'pads' in attrs
new_attr = make_attribute('auto_pad', 'VALID')
attrs[new_attr.name] = new_attr
elif legacy_pad_attr.i == caffe2_legacy_pb2.SAME:
assert not 'pads' in attrs
# default behavior in Caffe2 is SAME_UPPER
# https://github.com/caffe2/caffe2/blob/master/caffe2/operators/conv_pool_op_base.h#L39
new_attr = make_attribute('auto_pad', 'SAME_UPPER')
attrs[new_attr.name] = new_attr
elif legacy_pad_attr.i == caffe2_legacy_pb2.CAFFE_LEGACY_POOLING:
# The problem here is that, Pool op in Caffe may add an additional pixel, if the last part is smaller than stride.
# So we use the explicit padding to replace legacy_pad.
# pad[end] = output_size[start + 2] * stride[start] - pad[start] - 1 + kernel[start] - input[start + 2]
# end = start + len(pad) / 2
logger.warning('Converting legacy padding to explicit padding.')
for i in range(2):
attrs['pads'].ints[i + 2] = (output_size[i + 2] * attrs['strides'].ints[i] - attrs['pads'].ints[i]
- 1 + attrs['kernel_shape'].ints[i] - input_size[i + 2])
else:
logger.error('Don\'t know how to handle the legacy_pad, while processing operator:\n{}'.format(op_def))
raise
del node.attribute[:]
node.attribute.extend(attrs.values())
return node
def test_attr_repeated_float(self): # type: () -> None
attr = helper.make_attribute("floats", [1.0, 2.0])
self.assertEqual(attr.name, "floats")
self.assertEqual(list(attr.floats), [1.0, 2.0])
checker.check_attribute(attr)
def test_attr_repeated_int(self): # type: () -> None
attr = helper.make_attribute("ints", [1, 2])
self.assertEqual(attr.name, "ints")
self.assertEqual(list(attr.ints), [1, 2])
checker.check_attribute(attr)
def test_attr_repeated_str(self): # type: () -> None
attr = helper.make_attribute("strings", ["str1", "str2"])
self.assertEqual(attr.name, "strings")
self.assertEqual(list(attr.strings), [b"str1", b"str2"])
checker.check_attribute(attr)
