def apply_nhwc_to_nchw_permutation(graph: Graph):
# Add NHWC to NCHW permutation for all data nodes (only for nodes without permutation)
if graph.graph['layout'] == 'NCHW':
return
for node in graph.get_data_nodes():
if node.has_and_set('nchw_layout'):
continue
# Get NHWC to NCHW permutation for N dims, where N = len(node.shape)
permutation = PermuteAttrs().get_nhwc_to_nchw_permutation(
len(node.shape))
# Check that data node already has permutation
skip_permutation = False
for in_node in node.in_nodes():
edge_attrs = node.graph.get_edge_data(in_node.id, node.id)[0]
if 'permutation' in edge_attrs:
skip_permutation = True
for out_node in node.out_nodes():
edge_attrs = node.graph.get_edge_data(node.id, out_node.id)[0]
if 'permutation' in edge_attrs:
skip_permutation = True
if skip_permutation:
continue
# Set permutation to all in/out edges
for in_node in node.in_nodes():
PermuteAttrs.set_permutation(in_node, node, permutation)
for out_node in node.out_nodes():
PermuteAttrs.set_permutation(node, out_node, permutation)
def find_and_replace_pattern(self, graph: Graph):
for node in graph.get_data_nodes():
if node.has_and_set('nchw_layout'):
continue
# Get NHWC to NCHW permutation for N dims, where N = len(node.shape)
permutation = PermuteAttrs().get_nhwc_to_nchw_permutation(
len(node.shape))
# Check that data node already has permutation
skip_permutation = False
for in_node in node.in_nodes():
edge_attrs = node.graph.get_edge_data(in_node.id, node.id)[0]
if 'permutation' in edge_attrs:
skip_permutation = True
for out_node in node.out_nodes():
edge_attrs = node.graph.get_edge_data(node.id, out_node.id)[0]
if 'permutation' in edge_attrs:
skip_permutation = True
if skip_permutation:
continue
# Set permutation to all in/out edges
for in_node in node.in_nodes():
PermuteAttrs.set_permutation(in_node, node, permutation)
for out_node in node.out_nodes():
PermuteAttrs.set_permutation(node, out_node, permutation)
def conv_flatten_concat_action(graph: Graph, match: dict):
assert graph.graph['layout'] == 'NHWC'
reshape_node = match['reshape']
reshape_data_node = match['reshape_data']
conv_name = match['conv'].name
conv_data_node = match['conv_data']
# the pattern should be applied only in case when the reshape operation changes number of dimensions
if len(reshape_data_node.shape) == len(
conv_data_node.shape) or reshape_node.has_and_set('nchw_layout'):
return
if len(reshape_data_node.out_nodes()) == 1 and reshape_data_node.out_node().has_valid('type') and \
reshape_data_node.out_node().type == 'FullyConnected' and \
can_repack_fully_connected_weights_nhwc_to_nchw(reshape_data_node.out_node()):
log.info(
'There is a FullyConnected layer after the node "{}" which weights will be repacked. So there is no '
'need to insert Permute'.format(reshape_node.soft_get('name')))
return
graph.remove_edge(conv_data_node.id, reshape_node.id)
permutation_order = PermuteAttrs.get_nchw_to_nhwc_permutation(
len(conv_data_node.shape)).perm
new_permute_op = Permute(graph, {'order': permutation_order})
permute_data_node = new_permute_op.create_node_with_data(
[conv_data_node], dict(name=conv_name + '/Permute_'))
graph.create_edge(permute_data_node, reshape_node)
# Disable permutation for Reshape and Concat layers attributes
PermuteAttrs.set_permutation(reshape_node, reshape_data_node, None)
reshape_node['nchw_layout'] = True
def caffe_inner_product(node):
input_shape = node.in_node(0).shape
if input_shape is None:
return
batches = input_shape[0]
input_channels = np.prod(input_shape[1:])
if not node.has_valid('out-size'):
node['out-size'] = (np.prod(node.in_node(1).shape) /
input_channels).astype(np.int64)
output_channels = node['out-size']
weights_shape = np.array([output_channels, input_channels], dtype=np.int64)
# In case if original weight layout is IO we transpose them
if np.array_equal(node.in_node(1).shape, weights_shape[::-1]
) and node.soft_get('transpose_weights') is True:
node.in_node(1).value = np.transpose(node.in_node(1).value)
node.out_node().shape = np.array([batches, output_channels],
dtype=np.int64)
# Back propagation of shape to weights
node.in_node(1).shape = np.array(weights_shape)
node.in_node(1).value.shape = node.in_node(1).shape
mark_input_bins(node)
assign_dims_to_weights(node.in_node(1), None, 1, 0, 2)
PermuteAttrs.set_permutation(node.in_node(1), node, None)
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch):
# IE DetectionOutput layer consumes flattened confidences and locations tensors.
# That is why we add reshapes before them.
locs_node = match.single_input_node(0)
conf_node = match.single_input_node(1)
prior_boxes_node = match.single_input_node(2)
locs_out_nodes = locs_node[0].out_nodes()
assert len(locs_out_nodes) == 1
locs_out_node = locs_out_nodes[list(locs_out_nodes.keys())[0]]
assert locs_out_node.op == "OpOutput", locs_out_node.op
graph.remove_node(locs_out_node.id)
conf_out_nodes = conf_node[0].out_nodes()
assert len(conf_out_nodes) == 1
conf_out_node = conf_out_nodes[list(conf_out_nodes.keys())[0]]
assert conf_out_node.op == "OpOutput", conf_out_node.op
graph.remove_node(conf_out_node.id)
# reshape operation to flatten confidence tensor
reshape_loc_op = Reshape(graph, {'dim': np.array([0, -1])})
reshape_loc_node = reshape_loc_op.create_node(
[locs_node], dict(name='DetectionOutput_Reshape_loc_'))
# reshape operation to flatten confidence tensor
reshape_conf_op = Reshape(graph, {'dim': np.array([0, -1])})
reshape_conf_node = reshape_conf_op.create_node(
[conf_node], dict(name='DetectionOutput_Reshape_conf_'))
# remove the OpOutput node after the priors node
assert prior_boxes_node[0].out_node().op == "OpOutput"
graph.remove_node(prior_boxes_node[0].out_node().id)
# reshape operation for prior boxes tensor
reshape_priors_op = Reshape(graph, {'dim': np.array([1, 2, -1])})
reshape_priors_node = reshape_priors_op.create_node(
[prior_boxes_node], dict(name='DetectionOutput_Reshape_priors_'))
# create Detection Output node with three inputs: locations, confidences and prior boxes
detection_output_op = DetectionOutput(
graph, match.custom_replacement_desc.custom_attributes)
detection_output_node = detection_output_op.create_node(
[reshape_loc_node, reshape_conf_node, reshape_priors_node],
dict(name=detection_output_op.attrs['type'] + '_'))
PermuteAttrs.set_permutation(reshape_priors_node,
detection_output_node, None)
# create Output node to mark DetectionOutput as a graph output operation
output_op = Output(graph)
output_op.create_node([detection_output_node], dict(name='sink_'))
return {}
def replace_pattern(self, graph: Graph, match: dict):
if not match['input_0'].has_valid('value') and not match['input_1'].has_valid('value') or \
not match['input_0'].has_valid('value') and match['input_1'].has_valid('value') and match[
'input_1'].shape.size > 2:
match['fc']['type'] = 'GEMM'
elif not match['input_0'].has_valid('value') and match['input_1'].has_valid('value'):
match['fc']['type'] = 'FullyConnected'
node = match['fc']
mark_input_bins(node)
weights_node = match['input_1']
assign_dims_to_weights(weights_node, None, 0, 1, 2)
PermuteAttrs.set_permutation(weights_node, node, PermuteAttrs.Permutation(perm=int64_array([1, 0]),
inv=int64_array([0, 1])))
weights_shape = weights_node.shape
node['out-size'] = weights_shape[1]
def tf_matmul_infer(node):
assert (len(node.in_nodes()) == 2)
shapes = [node.in_node(i).shape.copy() for i in range(2)]
log.debug('matmul shapes: {}'.format(shapes))
if any(s is None or len(s) < 2 for s in shapes):
log.error("MatMul wasn't able to infer shape")
return
if node.transpose_a:
perm = np.array(range(len(node.in_node(0).shape)), dtype=np.int64)
perm[-1], perm[-2] = perm[-2], perm[-1]
inv = PermuteAttrs.get_inverse_permutation(perm)
permutation = PermuteAttrs.Permutation(perm=perm, inv=int64_array(inv))
PermuteAttrs.set_permutation(node.in_node(0), node, permutation)
shapes[0] = shapes[0][perm]
if node.transpose_b:
perm = np.array(range(len(node.in_node(1).shape)), dtype=np.int64)
perm[-1], perm[-2] = perm[-2], perm[-1]
inv = PermuteAttrs.get_inverse_permutation(perm)
permutation = PermuteAttrs.Permutation(perm=perm, inv=int64_array(inv))
PermuteAttrs.set_permutation(node.in_node(1), node, permutation)
shapes[1] = shapes[1][perm]
if any(shapes[0][:-2] != shapes[1][:-2]) or shapes[0][-1] != shapes[1][-2]:
log.error(
"MatMul wasn't able to infer shape because input dimensions are not compatible"
)
return
shape_tuple = (np.array(shapes[0][:-1], dtype=np.int64),
np.array([shapes[1][-1]], dtype=np.int64))
if len(shapes[0]) > 2:
# TODO Investigate case when MatMul have inputs with not matching output dimensions
# It looks to be a practical case and if we add outer dimensions of the first argument
# it will lead to incorrect model sometimes. TF documentation is unclear.
log.warning(
'Ignored outer dimensions of input tensor for MatMul node: {}'.
format(node.name))
# shape_tuple = (shapes[0][:-2], *shape_tuple)
log.debug('shape_tuple: {}'.format(shape_tuple))
node.out_node().shape = np.concatenate(shape_tuple)
node['channel_dims'] = node.out_node().shape.size - 1
log.debug('matmul shape: {}'.format(node.out_node().shape))
def infer(node: Node):
"""
Deconvolution has an input argument that explicitly determines output shape, so in contrast
to the forward Conv2d we shouldn't infer output shape. We just use this output shape as
an input shape and pass it to our utilities that computes numeric values for padding.
They also deliver output shape that is interpreted here as input shape for convolution.
We need to check that the real input shape and shape inferred by those utility functions match.
"""
output_shape = np.array(node.in_node(2).value)
batch = np.array(node.in_node(0).shape)[0]
output_shape[0] = batch
kernel_shape = node.in_node(1).shape
node['kernel_shape'] = kernel_shape
if output_shape is None or kernel_shape is None or node.spatial_dims is None or node.stride is None:
return
if not node.has_valid('kernel_spatial_idx'):
node['kernel_spatial_idx'] = np.delete(
[x for x in range(len(kernel_shape))],
(node.input_feature_channel, node.output_feature_channel))
if not node.has_valid('dilation'):
node['dilation'] = np.full([len(output_shape)], 1, dtype=np.int64)
spatial_dims = node.spatial_dims
output_spatial = np.array(output_shape[spatial_dims])
stride_spatial = np.array(node.stride[spatial_dims])
node['kernel_spatial'] = np.array(
kernel_shape[node.kernel_spatial_idx])
node.pad_spatial_shape, input_spatial_for_check = tf_window_op_pad_infer(
output_spatial, node.kernel_spatial, stride_spatial, node.auto_pad)
assert all(
input_spatial_for_check == node.in_node(0).shape[spatial_dims])
pad = np.zeros((len(output_shape), 2), dtype=np.int64)
pad[spatial_dims] = node.pad_spatial_shape
node.pad = pad
node.output = output_shape[node.channel_dims][0]
node.output_shape = output_shape
node.out_node().shape = output_shape
mark_input_bins(node, ['weights'], 1)
assign_dims_to_weights(node.in_node(1), node.kernel_spatial_idx,
node.input_feature_channel,
node.output_feature_channel, len(kernel_shape))
# OK, now we are sure this is a supported Deconvolution layer
node.type = 'Deconvolution'
node.op = 'Deconv2D'
# Add permute_attrs
PermuteAttrs.create_permute_attrs(
node,
attrs=[
('pad', 'input:0'),
('stride', 'input:0'),
('output_shape', 'input:0'),
('batch_dims', 'input:0'),
('channel_dims', 'input:0'),
('spatial_dims', 'input:0'),
('kernel_shape', 'input:1'),
('kernel_spatial_idx', 'input:1'),
('input_feature_channel', 'input:1'),
('output_feature_channel', 'input:1'),
])
PermuteAttrs.set_permutation(
node.in_node(1), node, node.get_weights_permute
if node.has_valid('get_weights_permute') else None)
PermuteInputs().set_input_permutation(node.in_node(2), node, 'input:0',
'shape')
node['force_precision_in_ports'] = {2: 'int64'}
def infer(node: Node):
"""
Infers shape of convolution node as it is done in ONNX.
It is very similar to one that Caffe does, but slightly different.
We made a complete fork of this function because they are supposed to be
supported differently by different people.
Args:
node: graph convolution node
"""
input_shape = node.in_node(0).shape
if input_shape is None:
return
# bias_term cannot be deduced earlier for frameworks that represent
# convolution weights/biases as regular inputs; so the number of inputs
# is being checked here and restore correct value for bias_term to
# have the rest of the code unchanged. It will be used after we merge
# several infer functions for convolution in different FWs to a single one.
if not node.has_valid('bias_term'):
node['bias_term'] = len(node.in_nodes()) == 3
weights_index = node.weights_index if node.has_valid(
'weights_index') else 1
# Reshape weights kernel to original shape
# In case of caffe ot MXNet framework, values for weights has no structed shape like OIHW
# so we have to reshape weights to normal shape
# For this case, Convolution node should have attribute reshape_kernel = True
if node.has_valid('reshape_kernel') and node.reshape_kernel:
if not (node.has_valid('output') and node.has_valid('channel_dims')
and node.has_valid('group')
and node.has_valid('kernel_spatial')):
log.error(
'Cannot reshape kernel due to not all required attrs was set to {} node'
.format(node.id))
return
# layout for Convolution weights is OIHW
kernel_shape = np.array([
node.output,
input_shape[node.channel_dims].item() / node.group, *[
node.kernel_spatial[i]
for i in range(len(node.kernel_spatial))
]
],
dtype=np.int64)
if node.type == 'Deconvolution': # layout for Deconvolution weights is IOHW
kernel_shape[[0, 1]] = kernel_shape[[1, 0]]
#node.input_feature_channel, node.output_feature_channel = node.output_feature_channel, node.input_feature_channel
if np.prod(kernel_shape) != np.prod(
node.in_node(weights_index).value.shape):
log.error(
"Size of weights {} does not match kernel shape: {}\n".
format(np.prod(node.in_node(weights_index).value.shape),
kernel_shape) +
" Possible reason is wrong channel number in input shape\n"
)
raise Error("Cannot reshape weights to kernel shape")
node.in_node(weights_index).shape = np.array(kernel_shape)
node.in_node(weights_index).value = np.reshape(
node.in_node(weights_index).value, kernel_shape)
node.reshape_kernel = False
# Pass weights shape to node attribute kernel_shape
kernel_shape = node.in_node(weights_index).shape
node['kernel_shape'] = kernel_shape
# Calculate kernel_spatial_idx and spatial_dims if it is not specified
# It is necessary for ONNX dut to convolution can be 1D/2D/3D
if not node.has_valid('kernel_spatial_idx'):
node['kernel_spatial_idx'] = np.delete(
[x for x in range(len(kernel_shape))],
(node.input_feature_channel, node.output_feature_channel))
if not node.has_valid('spatial_dims'):
node['spatial_dims'] = np.delete(
[x for x in range(len(input_shape))],
(node.channel_dims[0], node.batch_dims[0]))
node['kernel_spatial'] = kernel_shape[node.kernel_spatial_idx]
if not node.has_valid('output'):
# restore the number of output feature maps from the second argument that is weights
if node.type in [
'Convolution', 'Deconvolution', 'DeformableConvolution'
]:
node['output'] = kernel_shape[node.output_feature_channel]
else:
raise Error(
'Convolution infer function was called for a node {} with unsupported type {}',
node.soft_get('name'), node.type)
# Set default values for dilation, strides and pads if not set
if not node.has_valid('dilation'):
node['dilation'] = np.full([len(input_shape)], 1, dtype=np.int64)
if not node.has_valid('stride'):
node['stride'] = np.full([len(input_shape)], 1, dtype=np.int64)
if not node.has_valid('pad'):
node['pad'] = np.array([[0, 0]] * len(input_shape), dtype=np.int64)
node['pad_spatial_shape'] = node.pad[node.spatial_dims]
if not node.has_valid('output_padding'):
node['output_padding'] = np.full([len(input_shape)],
0,
dtype=np.int64)
input_spatial_shape = input_shape[node.spatial_dims]
stride_spatial_shape = node.stride[node.spatial_dims]
kernel_extent = node.dilation[node.spatial_dims] * (
node.kernel_spatial - 1) + 1
# TensorFlow always has auto_pad attribute that can be either valid or same_upper
# In ONNX auto_pad attribute is deprecated but appears in some models (could be valid, same_upper or same_lower)
# Caffe do not use auto_pad attribute
if node.has_valid(
'auto_pad') and not node.has_valid('output_spatial_shape'):
node['pad_spatial_shape'], node[
'output_spatial_shape'] = tf_window_op_pad_infer(
input_spatial_shape, kernel_extent, stride_spatial_shape,
node.auto_pad, node.type == 'Deconvolution')
pad = np.zeros((len(input_shape), 2), dtype=np.int64)
pad[node.spatial_dims] = node.pad_spatial_shape
node.pad = pad
else:
pad_spatial_shape = np.add.reduce(node.pad_spatial_shape, axis=1)
if node.type == 'Convolution':
float_spatial = Convolution.calc_convolution(
input_spatial_shape, stride_spatial_shape,
pad_spatial_shape, kernel_extent)
node['output_spatial_shape'] = int64_array(float_spatial)
elif node.type == 'Deconvolution':
# In case of given output_spatial_shape we calculate pads spatial
if node.has_valid('output_spatial_shape'):
if node.has_valid('get_pad'):
node['pad'] = node.get_pad(node, input_shape,
kernel_shape)
else:
log.debug(
'Can\'t calculate paddings due to missing lambda get_pad in {} node'
.format(node.id))
return
else:
output_padding = node.output_padding[
node.spatial_dims] if node.has_valid(
'output_padding') else None
if output_padding is not None and any(output_padding):
pad_spatial_shape -= output_padding
for dim in range(len(pad_spatial_shape)):
node.pad_spatial_shape[dim][
1] -= pad_spatial_shape[dim]
node.pad[node.spatial_dims] = node.pad_spatial_shape
node['output_padding'] = None
float_spatial = Convolution.calc_deconvolution(
node, input_spatial_shape, pad_spatial_shape,
kernel_extent)
node['output_spatial_shape'] = int64_array(float_spatial)
elif node.type == 'DeformableConvolution':
# get the output spatial shape from the second input with offsets
node['output_spatial_shape'] = int64_array(
[node.in_node(1).shape[2:4]])
else:
assert 'Unsupported layer type "{}"'.format(node.type)
# For cases when group attribute wasn't set in extractor we should specify get_group attribute
# this attribute should store lambda node: ... (check tf convolution extractor)
if node.has_valid('get_group'):
node['group'] = node.get_group(node)
output_shape = np.full_like(input_shape, -1, dtype=np.int64)
output_shape[node.batch_dims] = input_shape[node.batch_dims] # pylint: disable=unsupported-assignment-operation
output_shape[node.spatial_dims] = node.output_spatial_shape # pylint: disable=unsupported-assignment-operation
# For cases when output attribute wasn't set in extractor we should specify get_output_feature_dim attribute
# this attribute should store lambda node: ... (check tf convolution extractor)
if node.has_valid('get_output_feature_dim'):
node['output'] = node.get_output_feature_dim(node)
output_shape[node.channel_dims] = node.output # pylint: disable=unsupported-assignment-operation
node['output_shape'] = output_shape
for n in node.out_nodes():
node.out_node(n).shape = output_shape
mark_input_bins(
node, start_port=1 if node.type != 'DeformableConvolution' else 2)
assign_dims_to_weights(node.in_node(weights_index),
node.kernel_spatial_idx,
node.input_feature_channel,
node.output_feature_channel, len(kernel_shape))
PermuteAttrs.create_permute_attrs(
node,
attrs=[
('pad', 'input:0'),
('stride', 'input:0'),
('dilation', 'input:0'),
('output_shape', 'input:0'),
('batch_dims', 'input:0'),
('channel_dims', 'input:0'),
('spatial_dims', 'input:0'),
('kernel_shape', 'input:{}'.format(weights_index)),
('kernel_spatial_idx', 'input:{}'.format(weights_index)),
('input_feature_channel', 'input:{}'.format(weights_index)),
('output_feature_channel', 'input:{}'.format(weights_index)),
])
PermuteAttrs.set_permutation(
node.in_node(weights_index), node, node.get_weights_permute
if node.has_valid('get_weights_permute') else None)
def infer(node: Node):
"""
Deconvolution has an input argument that explicitly determines output shape, so in contrast
to the forward Conv2d we shouldn't infer output shape. We just use this output shape as
an input shape and pass it to our utilities that computes numeric values for padding.
They also deliver output shape that is interpreted here as input shape for convolution.
We need to check that the real input shape and shape inferred by those utility functions match.
"""
output_shape = shape_array(node.in_node(2).value)
output_shape[0] = node.in_port(0).data.get_shape()[0]
kernel_shape = node.in_port(1).data.get_shape()
node['kernel_shape'] = kernel_shape
if output_shape is None or kernel_shape is None or node.spatial_dims is None or node.stride is None:
return
if not node.has_valid('kernel_spatial_idx'):
node['kernel_spatial_idx'] = np.delete(
[x for x in range(len(kernel_shape))],
(node.input_feature_channel, node.output_feature_channel))
if not node.has_valid('dilation'):
node['dilation'] = np.full([len(output_shape)], 1, dtype=np.int64)
spatial_dims = node.spatial_dims
output_spatial = shape_array(output_shape[spatial_dims])
stride_spatial = shape_array(node.stride[spatial_dims])
node['kernel_spatial'] = shape_array(
kernel_shape[node.kernel_spatial_idx])
node.pad_spatial_shape, input_spatial_for_check = tf_window_op_pad_infer(
output_spatial, node.kernel_spatial, stride_spatial, node.auto_pad)
assert compatible_shapes(input_spatial_for_check,
node.in_node(0).shape[spatial_dims])
pad = np.zeros((len(output_shape), 2), dtype=np.int64)
pad[spatial_dims] = node.pad_spatial_shape
node.pad = pad
node.output = output_shape[node.channel_dims][0]
node.output_shape = output_shape
node.out_port(0).data.set_shape(output_shape)
mark_input_bins(node, ['weights'], 1)
assign_dims_to_weights(node.in_node(1), node.kernel_spatial_idx,
node.input_feature_channel,
node.output_feature_channel, len(kernel_shape))
# OK, now we are sure this is a supported Deconvolution layer
node.type = 'Deconvolution'
node.op = 'Deconv2D'
# Add permute_attrs
PermuteAttrs.create_permute_attrs(
node,
attrs=[
('pad', 'input:0'),
('stride', 'input:0'),
('output_shape', 'input:0'),
('batch_dims', 'input:0'),
('channel_dims', 'input:0'),
('spatial_dims', 'input:0'),
('kernel_shape', 'input:1'),
('kernel_spatial_idx', 'input:1'),
('input_feature_channel', 'input:1'),
('output_feature_channel', 'input:1'),
])
# is needed to permute Deconv weights from the original TF [H, W, C_OUT, C_IN] into IE [C_IN, C_OUT, H, W]
# but for other nodes in weights subgraph permutations must turned off
# by marking with MarkSubGraphsWithCorrectLayout even if graph layout is NCHW.
PermuteAttrs.set_permutation(
node.in_node(1), node, node.soft_get('get_weights_permute', None))
PermuteInputs().set_input_permutation(node.in_node(1), node, 'input:1',
'transpose')
PermuteInputs().set_input_permutation(node.in_node(2), node, 'input:0',
'shape')
node['force_precision_in_ports'] = {2: 'int64'}
