def infer(node: Node):
"""
Performs shape inference of MatMul node as operation doc-string says
Raises on any shape inconsistency
"""
name = node.soft_get('name', str(node.id))
connected_in_ports = {
idx: port
for idx, port in node.in_ports().items()
if not port.disconnected()
}
assert len(connected_in_ports) == 2 and 0 in connected_in_ports and 1 in connected_in_ports, \
"MatMul should have 2 connected input ports, but it doesn't for node: `{}`. Ports: {}" \
"".format(name, connected_in_ports)
log.debug('MatMul `{}` input shapes: {}'.format(
name, [node.in_port(i).data.get_shape() for i in range(2)]))
A_shape, B_shape = MatMul.shape_alignment(node)
log.debug('MatMul `{}` aligned input shapes: {}'.format(
name, [A_shape, B_shape]))
assert A_shape[-1] == B_shape[-2], \
"MatMul input shapes are incorrect. COL_INDEX_DIMs are not equal. Node: {}. Shapes: {}" \
"".format(name, [A_shape, B_shape])
output_shape = np.concatenate((A_shape[:-1], B_shape[-1:]))
node.out_port(0).data.set_shape(output_shape)
in_ch = 0 if not node.transpose_b else 1
out_ch = 1 if not node.transpose_b else 0
assign_dims_to_weights(node.in_node(1), None, in_ch, out_ch,
node.in_port(1).data.get_shape().size)
MatMul.value_propagation(node)
def caffe_inner_product(node):
input_shape = node.in_node(0).shape
if input_shape is None:
return
batches = input_shape[0]
input_channels = np.prod(input_shape[1:])
if not node.has_valid('out-size'):
node['out-size'] = (np.prod(node.in_node(1).shape) /
input_channels).astype(np.int64)
output_channels = node['out-size']
weights_shape = np.array([output_channels, input_channels], dtype=np.int64)
# In case if original weight layout is IO we transpose them
if np.array_equal(node.in_node(1).shape, weights_shape[::-1]
) and node.soft_get('transpose_weights') is True:
node.in_node(1).value = np.transpose(node.in_node(1).value)
node.out_node().shape = np.array([batches, output_channels],
dtype=np.int64)
# Back propagation of shape to weights
node.in_node(1).shape = np.array(weights_shape)
node.in_node(1).value.shape = node.in_node(1).shape
mark_input_bins(node)
assign_dims_to_weights(node.in_node(1), None, 1, 0, 2)
PermuteAttrs.set_permutation(node.in_node(1), node, None)
def matmul_to_fully_connected_action(graph: Graph, match: dict):
log.debug('fully_connected_matched')
matmul = match['matmul']
input = matmul.in_node(0)
weights = matmul.in_node(1)
weights_consumers = graph.out_edges(weights.node)
log.debug(
"len(input.shape) = {}, len(weights.shape) = {}, len(weights_consumers) = {}"
.format(
len(input.shape) if input.shape is not None else None,
len(weights.shape) if not None else None,
len(weights_consumers) if weights_consumers is not None else None))
if not (weights.value is not None and input.shape is not None
and len(input.shape) >= 2 and weights.shape is not None
and len(weights.shape) == 2 and len(weights_consumers) >= 1):
matmul['can_be_fused'] = False
return
matmul['out-size'] = matmul.out_node().shape[-1]
if graph.graph['cmd_params'].generate_experimental_IR_V10:
matmul['type'] = 'MatMul'
else:
matmul['type'] = 'FullyConnected'
matmul.in_edge(1)['bin'] = 'weights'
matmul['channel_dims'] = len(match['output'].shape) - 1
matmul['bias_addable'] = True
matmul['input_channel_dim'] = 0 # MatMul weights in IO
matmul['output_channel_dim'] = 1
matmul['layout'] = 'NHWC'
assign_dims_to_weights(matmul.in_node(1), None, matmul.input_channel_dim,
matmul.output_channel_dim, 2)
def gemm_to_fully_connected_action(graph: nx.MultiDiGraph, match: dict):
log.debug('gemm_to_fully_connected_action is triggered')
gemm = match['gemm']
A = gemm.in_node(0)
B = gemm.in_node(1)
B_consumers = graph.out_edges(B.node)
C = gemm.in_node(2)
C_consumers = graph.out_edges(C.node)
if not (B.value is not None and C.value is not None and A.shape is not None
and C.shape.size == 1 and not gemm.transpose_a and
(len(B_consumers) == 1 or not gemm.transpose_b)):
log.warning('Cannot convert Gemm to FullyConnected')
return
if gemm.transpose_b:
# B.value = B.value.transpose()
# B.shape = np.array(B.value.shape, dtype=np.int64)
gemm.transpose_b = 0
else:
B.value = B.value.transpose()
B.shape = np.array(B.value.shape, dtype=np.int64)
gemm['out-size'] = gemm.out_node().shape[-1]
gemm['type'] = 'FullyConnected'
gemm['channel_dims'] = len(match['output'].shape) - 1
gemm['bias_addable'] = True
gemm['input_channel_dim'] = 1 # MatMul weights in IO
gemm['output_channel_dim'] = 0
gemm['layout'] = 'NCHW'
gemm.in_edge(1)['bin'] = 'weights'
gemm.in_edge(2)['bin'] = 'biases'
assign_dims_to_weights(gemm.in_node(1), None, 1, 0, 2)
def replace_pattern(self, graph: Graph, match: dict):
if not match['input_0'].has_valid('value') and not match['input_1'].has_valid('value') or \
not match['input_0'].has_valid('value') and match['input_1'].has_valid('value') and match[
'input_1'].shape.size > 2:
match['fc']['type'] = 'GEMM'
elif not match['input_0'].has_valid('value') and match['input_1'].has_valid('value'):
match['fc']['type'] = 'FullyConnected'
node = match['fc']
mark_input_bins(node)
weights_node = match['input_1']
assign_dims_to_weights(weights_node, None, 0, 1, 2)
PermuteAttrs.set_permutation(weights_node, node, PermuteAttrs.Permutation(perm=int64_array([1, 0]),
inv=int64_array([0, 1])))
weights_shape = weights_node.shape
node['out-size'] = weights_shape[1]
def replace_pattern(self, graph: Graph, match: Dict[str, Node]):
log.debug('GemmToFullyConnected is triggered')
gemm = match['gemm']
A = gemm.in_node(0)
B = gemm.in_node(1)
B_consumers = graph.out_edges(B.node)
C = gemm.in_node(2)
if not (B.value is not None and C.value is not None
and A.shape is not None and not gemm.transpose_a and
(len(B_consumers) == 1 or not gemm.transpose_b)):
log.warning('Cannot convert Gemm to FullyConnected')
return
if gemm.transpose_b:
# B.value = B.value.transpose()
# B.shape = np.array(B.value.shape, dtype=np.int64)
gemm.transpose_b = 0
else:
B.value = B.value.transpose()
B.shape = np.array(B.value.shape, dtype=np.int64)
gemm['out-size'] = gemm.out_port(0).data.get_shape()[-1]
gemm['type'] = 'FullyConnected'
gemm['channel_dims'] = len(match['output'].shape) - 1
gemm['bias_addable'] = True
gemm['input_channel_dim'] = 1 # MatMul weights in IO
gemm['output_channel_dim'] = 0
gemm['layout'] = 'NCHW'
gemm.in_port(1).bin = 'weights'
bias_node = Add(graph, {}).create_node()
gemm.out_port(0).get_connection().set_source(bias_node.out_port(0))
gemm.in_port(2).get_connection().set_destination(bias_node.in_port(1))
gemm.out_port(0).connect(bias_node.in_port(0))
assign_dims_to_weights(gemm.in_node(1), None, 1, 0, 2)
def infer(node: Node):
"""
Deconvolution has an input argument that explicitly determines output shape, so in contrast
to the forward Conv2d we shouldn't infer output shape. We just use this output shape as
an input shape and pass it to our utilities that computes numeric values for padding.
They also deliver output shape that is interpreted here as input shape for convolution.
We need to check that the real input shape and shape inferred by those utility functions match.
"""
output_shape = np.array(node.in_node(2).value)
batch = np.array(node.in_node(0).shape)[0]
output_shape[0] = batch
kernel_shape = node.in_node(1).shape
node['kernel_shape'] = kernel_shape
if output_shape is None or kernel_shape is None or node.spatial_dims is None or node.stride is None:
return
if not node.has_valid('kernel_spatial_idx'):
node['kernel_spatial_idx'] = np.delete(
[x for x in range(len(kernel_shape))],
(node.input_feature_channel, node.output_feature_channel))
if not node.has_valid('dilation'):
node['dilation'] = np.full([len(output_shape)], 1, dtype=np.int64)
spatial_dims = node.spatial_dims
output_spatial = np.array(output_shape[spatial_dims])
stride_spatial = np.array(node.stride[spatial_dims])
node['kernel_spatial'] = np.array(
kernel_shape[node.kernel_spatial_idx])
node.pad_spatial_shape, input_spatial_for_check = tf_window_op_pad_infer(
output_spatial, node.kernel_spatial, stride_spatial, node.auto_pad)
assert all(
input_spatial_for_check == node.in_node(0).shape[spatial_dims])
pad = np.zeros((len(output_shape), 2), dtype=np.int64)
pad[spatial_dims] = node.pad_spatial_shape
node.pad = pad
node.output = output_shape[node.channel_dims][0]
node.output_shape = output_shape
node.out_node().shape = output_shape
mark_input_bins(node, ['weights'], 1)
assign_dims_to_weights(node.in_node(1), node.kernel_spatial_idx,
node.input_feature_channel,
node.output_feature_channel, len(kernel_shape))
# OK, now we are sure this is a supported Deconvolution layer
node.type = 'Deconvolution'
node.op = 'Deconv2D'
# Add permute_attrs
PermuteAttrs.create_permute_attrs(
node,
attrs=[
('pad', 'input:0'),
('stride', 'input:0'),
('output_shape', 'input:0'),
('batch_dims', 'input:0'),
('channel_dims', 'input:0'),
('spatial_dims', 'input:0'),
('kernel_shape', 'input:1'),
('kernel_spatial_idx', 'input:1'),
('input_feature_channel', 'input:1'),
('output_feature_channel', 'input:1'),
])
PermuteAttrs.set_permutation(
node.in_node(1), node, node.get_weights_permute
if node.has_valid('get_weights_permute') else None)
PermuteInputs().set_input_permutation(node.in_node(2), node, 'input:0',
'shape')
node['force_precision_in_ports'] = {2: 'int64'}
def infer(node: Node):
"""
Infers shape of convolution node as it is done in ONNX.
It is very similar to one that Caffe does, but slightly different.
We made a complete fork of this function because they are supposed to be
supported differently by different people.
Args:
node: graph convolution node
"""
input_shape = node.in_node(0).shape
if input_shape is None:
return
# bias_term cannot be deduced earlier for frameworks that represent
# convolution weights/biases as regular inputs; so the number of inputs
# is being checked here and restore correct value for bias_term to
# have the rest of the code unchanged. It will be used after we merge
# several infer functions for convolution in different FWs to a single one.
if not node.has_valid('bias_term'):
node['bias_term'] = len(node.in_nodes()) == 3
weights_index = node.weights_index if node.has_valid(
'weights_index') else 1
# Reshape weights kernel to original shape
# In case of caffe ot MXNet framework, values for weights has no structed shape like OIHW
# so we have to reshape weights to normal shape
# For this case, Convolution node should have attribute reshape_kernel = True
if node.has_valid('reshape_kernel') and node.reshape_kernel:
if not (node.has_valid('output') and node.has_valid('channel_dims')
and node.has_valid('group')
and node.has_valid('kernel_spatial')):
log.error(
'Cannot reshape kernel due to not all required attrs was set to {} node'
.format(node.id))
return
# layout for Convolution weights is OIHW
kernel_shape = np.array([
node.output,
input_shape[node.channel_dims].item() / node.group, *[
node.kernel_spatial[i]
for i in range(len(node.kernel_spatial))
]
],
dtype=np.int64)
if node.type == 'Deconvolution': # layout for Deconvolution weights is IOHW
kernel_shape[[0, 1]] = kernel_shape[[1, 0]]
#node.input_feature_channel, node.output_feature_channel = node.output_feature_channel, node.input_feature_channel
if np.prod(kernel_shape) != np.prod(
node.in_node(weights_index).value.shape):
log.error(
"Size of weights {} does not match kernel shape: {}\n".
format(np.prod(node.in_node(weights_index).value.shape),
kernel_shape) +
" Possible reason is wrong channel number in input shape\n"
)
raise Error("Cannot reshape weights to kernel shape")
node.in_node(weights_index).shape = np.array(kernel_shape)
node.in_node(weights_index).value = np.reshape(
node.in_node(weights_index).value, kernel_shape)
node.reshape_kernel = False
# Pass weights shape to node attribute kernel_shape
kernel_shape = node.in_node(weights_index).shape
node['kernel_shape'] = kernel_shape
# Calculate kernel_spatial_idx and spatial_dims if it is not specified
# It is necessary for ONNX dut to convolution can be 1D/2D/3D
if not node.has_valid('kernel_spatial_idx'):
node['kernel_spatial_idx'] = np.delete(
[x for x in range(len(kernel_shape))],
(node.input_feature_channel, node.output_feature_channel))
if not node.has_valid('spatial_dims'):
node['spatial_dims'] = np.delete(
[x for x in range(len(input_shape))],
(node.channel_dims[0], node.batch_dims[0]))
node['kernel_spatial'] = kernel_shape[node.kernel_spatial_idx]
if not node.has_valid('output'):
# restore the number of output feature maps from the second argument that is weights
if node.type in [
'Convolution', 'Deconvolution', 'DeformableConvolution'
]:
node['output'] = kernel_shape[node.output_feature_channel]
else:
raise Error(
'Convolution infer function was called for a node {} with unsupported type {}',
node.soft_get('name'), node.type)
# Set default values for dilation, strides and pads if not set
if not node.has_valid('dilation'):
node['dilation'] = np.full([len(input_shape)], 1, dtype=np.int64)
if not node.has_valid('stride'):
node['stride'] = np.full([len(input_shape)], 1, dtype=np.int64)
if not node.has_valid('pad'):
node['pad'] = np.array([[0, 0]] * len(input_shape), dtype=np.int64)
node['pad_spatial_shape'] = node.pad[node.spatial_dims]
if not node.has_valid('output_padding'):
node['output_padding'] = np.full([len(input_shape)],
0,
dtype=np.int64)
input_spatial_shape = input_shape[node.spatial_dims]
stride_spatial_shape = node.stride[node.spatial_dims]
kernel_extent = node.dilation[node.spatial_dims] * (
node.kernel_spatial - 1) + 1
# TensorFlow always has auto_pad attribute that can be either valid or same_upper
# In ONNX auto_pad attribute is deprecated but appears in some models (could be valid, same_upper or same_lower)
# Caffe do not use auto_pad attribute
if node.has_valid(
'auto_pad') and not node.has_valid('output_spatial_shape'):
node['pad_spatial_shape'], node[
'output_spatial_shape'] = tf_window_op_pad_infer(
input_spatial_shape, kernel_extent, stride_spatial_shape,
node.auto_pad, node.type == 'Deconvolution')
pad = np.zeros((len(input_shape), 2), dtype=np.int64)
pad[node.spatial_dims] = node.pad_spatial_shape
node.pad = pad
else:
pad_spatial_shape = np.add.reduce(node.pad_spatial_shape, axis=1)
if node.type == 'Convolution':
float_spatial = Convolution.calc_convolution(
input_spatial_shape, stride_spatial_shape,
pad_spatial_shape, kernel_extent)
node['output_spatial_shape'] = int64_array(float_spatial)
elif node.type == 'Deconvolution':
# In case of given output_spatial_shape we calculate pads spatial
if node.has_valid('output_spatial_shape'):
if node.has_valid('get_pad'):
node['pad'] = node.get_pad(node, input_shape,
kernel_shape)
else:
log.debug(
'Can\'t calculate paddings due to missing lambda get_pad in {} node'
.format(node.id))
return
else:
output_padding = node.output_padding[
node.spatial_dims] if node.has_valid(
'output_padding') else None
if output_padding is not None and any(output_padding):
pad_spatial_shape -= output_padding
for dim in range(len(pad_spatial_shape)):
node.pad_spatial_shape[dim][
1] -= pad_spatial_shape[dim]
node.pad[node.spatial_dims] = node.pad_spatial_shape
node['output_padding'] = None
float_spatial = Convolution.calc_deconvolution(
node, input_spatial_shape, pad_spatial_shape,
kernel_extent)
node['output_spatial_shape'] = int64_array(float_spatial)
elif node.type == 'DeformableConvolution':
# get the output spatial shape from the second input with offsets
node['output_spatial_shape'] = int64_array(
[node.in_node(1).shape[2:4]])
else:
assert 'Unsupported layer type "{}"'.format(node.type)
# For cases when group attribute wasn't set in extractor we should specify get_group attribute
# this attribute should store lambda node: ... (check tf convolution extractor)
if node.has_valid('get_group'):
node['group'] = node.get_group(node)
output_shape = np.full_like(input_shape, -1, dtype=np.int64)
output_shape[node.batch_dims] = input_shape[node.batch_dims] # pylint: disable=unsupported-assignment-operation
output_shape[node.spatial_dims] = node.output_spatial_shape # pylint: disable=unsupported-assignment-operation
# For cases when output attribute wasn't set in extractor we should specify get_output_feature_dim attribute
# this attribute should store lambda node: ... (check tf convolution extractor)
if node.has_valid('get_output_feature_dim'):
node['output'] = node.get_output_feature_dim(node)
output_shape[node.channel_dims] = node.output # pylint: disable=unsupported-assignment-operation
node['output_shape'] = output_shape
for n in node.out_nodes():
node.out_node(n).shape = output_shape
mark_input_bins(
node, start_port=1 if node.type != 'DeformableConvolution' else 2)
assign_dims_to_weights(node.in_node(weights_index),
node.kernel_spatial_idx,
node.input_feature_channel,
node.output_feature_channel, len(kernel_shape))
PermuteAttrs.create_permute_attrs(
node,
attrs=[
('pad', 'input:0'),
('stride', 'input:0'),
('dilation', 'input:0'),
('output_shape', 'input:0'),
('batch_dims', 'input:0'),
('channel_dims', 'input:0'),
('spatial_dims', 'input:0'),
('kernel_shape', 'input:{}'.format(weights_index)),
('kernel_spatial_idx', 'input:{}'.format(weights_index)),
('input_feature_channel', 'input:{}'.format(weights_index)),
('output_feature_channel', 'input:{}'.format(weights_index)),
])
PermuteAttrs.set_permutation(
node.in_node(weights_index), node, node.get_weights_permute
if node.has_valid('get_weights_permute') else None)
def infer(node: Node):
"""
Deconvolution has an input argument that explicitly determines output shape, so in contrast
to the forward Conv2d we shouldn't infer output shape. We just use this output shape as
an input shape and pass it to our utilities that computes numeric values for padding.
They also deliver output shape that is interpreted here as input shape for convolution.
We need to check that the real input shape and shape inferred by those utility functions match.
"""
output_shape = shape_array(node.in_node(2).value)
output_shape[0] = node.in_port(0).data.get_shape()[0]
kernel_shape = node.in_port(1).data.get_shape()
node['kernel_shape'] = kernel_shape
if output_shape is None or kernel_shape is None or node.spatial_dims is None or node.stride is None:
return
if not node.has_valid('kernel_spatial_idx'):
node['kernel_spatial_idx'] = np.delete(
[x for x in range(len(kernel_shape))],
(node.input_feature_channel, node.output_feature_channel))
if not node.has_valid('dilation'):
node['dilation'] = np.full([len(output_shape)], 1, dtype=np.int64)
spatial_dims = node.spatial_dims
output_spatial = shape_array(output_shape[spatial_dims])
stride_spatial = shape_array(node.stride[spatial_dims])
node['kernel_spatial'] = shape_array(
kernel_shape[node.kernel_spatial_idx])
node.pad_spatial_shape, input_spatial_for_check = tf_window_op_pad_infer(
output_spatial, node.kernel_spatial, stride_spatial, node.auto_pad)
assert compatible_shapes(input_spatial_for_check,
node.in_node(0).shape[spatial_dims])
pad = np.zeros((len(output_shape), 2), dtype=np.int64)
pad[spatial_dims] = node.pad_spatial_shape
node.pad = pad
node.output = output_shape[node.channel_dims][0]
node.output_shape = output_shape
node.out_port(0).data.set_shape(output_shape)
mark_input_bins(node, ['weights'], 1)
assign_dims_to_weights(node.in_node(1), node.kernel_spatial_idx,
node.input_feature_channel,
node.output_feature_channel, len(kernel_shape))
# OK, now we are sure this is a supported Deconvolution layer
node.type = 'Deconvolution'
node.op = 'Deconv2D'
# Add permute_attrs
PermuteAttrs.create_permute_attrs(
node,
attrs=[
('pad', 'input:0'),
('stride', 'input:0'),
('output_shape', 'input:0'),
('batch_dims', 'input:0'),
('channel_dims', 'input:0'),
('spatial_dims', 'input:0'),
('kernel_shape', 'input:1'),
('kernel_spatial_idx', 'input:1'),
('input_feature_channel', 'input:1'),
('output_feature_channel', 'input:1'),
])
# is needed to permute Deconv weights from the original TF [H, W, C_OUT, C_IN] into IE [C_IN, C_OUT, H, W]
# but for other nodes in weights subgraph permutations must turned off
# by marking with MarkSubGraphsWithCorrectLayout even if graph layout is NCHW.
PermuteAttrs.set_permutation(
node.in_node(1), node, node.soft_get('get_weights_permute', None))
PermuteInputs().set_input_permutation(node.in_node(1), node, 'input:1',
'transpose')
PermuteInputs().set_input_permutation(node.in_node(2), node, 'input:0',
'shape')
node['force_precision_in_ports'] = {2: 'int64'}
def replace_pattern(self, graph: Graph, match: Dict[str, Node]):
log.debug('GemmToFullyConnected is triggered')
gemm = match['gemm']
# TODO nGraph remove BEGIN
if not graph.graph['cmd_params'].generate_experimental_IR_V10:
A = gemm.in_node(0)
B = gemm.in_node(1)
B_consumers = graph.out_edges(B.node)
C = gemm.in_node(2)
if not (B.value is not None and C.value is not None
and A.shape is not None and not gemm.transpose_a and
(len(B_consumers) == 1 or not gemm.transpose_b)):
log.warning('Cannot convert Gemm to FullyConnected')
return
if gemm.transpose_b:
# B.value = B.value.transpose()
# B.shape = np.array(B.value.shape, dtype=np.int64)
gemm.transpose_b = 0
else:
B.value = B.value.transpose()
B.shape = np.array(B.value.shape, dtype=np.int64)
gemm['out-size'] = gemm.out_port(0).data.get_shape()[-1]
gemm['type'] = 'FullyConnected'
gemm['channel_dims'] = len(match['output'].shape) - 1
gemm['bias_addable'] = True
gemm['input_channel_dim'] = 1 # MatMul weights in IO
gemm['output_channel_dim'] = 0
gemm['layout'] = 'NCHW'
gemm.in_port(1).bin = 'weights'
else:
B = gemm.in_node(1)
assert B.value is not None
if gemm.transpose_b:
B.value = B.value.transpose()
B.shape = np.array(B.value.shape, dtype=np.int64)
bias_node = Add(graph, {'name': 'MatMulBias_'}).create_node()
gemm.out_port(0).get_connection().set_source(bias_node.out_port(0))
gemm.in_port(2).get_connection().set_destination(bias_node.in_port(1))
gemm.out_port(0).connect(bias_node.in_port(0))
if graph.graph['cmd_params'].generate_experimental_IR_V10:
gemm.type = 'MatMul'
if gemm.has_valid('alpha'):
if not math.isclose(gemm.alpha, 1):
mul_node = Mul(graph, {'name': 'MatMulAlpha_'}).create_node()
const = Const(graph, {
'value': np.array(gemm.alpha)
}).create_node()
bias_node.in_port(0).get_connection().set_destination(
mul_node.in_port(0))
bias_node.in_port(0).connect(mul_node.out_port(0))
mul_node.in_port(1).connect(const.out_port(0))
del gemm['alpha']
if gemm.has_valid('beta'):
if not math.isclose(gemm.beta, 1):
mul_node = Mul(graph, {'name': 'MatMulBeta_'}).create_node()
const = Const(graph, {
'value': np.array(gemm.beta)
}).create_node()
bias_node.in_port(1).get_connection().set_destination(
mul_node.in_port(0))
bias_node.in_port(1).connect(mul_node.out_port(0))
mul_node.in_port(1).connect(const.out_port(0))
del gemm['beta']
if not graph.graph['cmd_params'].generate_experimental_IR_V10:
assign_dims_to_weights(gemm.in_node(1), None, 1, 0, 2)
