def generate_sub_graph(self, graph: Graph, match: SubgraphMatch):
# IE DetectionOutput layer consumes flattened confidences and locations tensors.
# That is why we add reshapes before them.
locs_node = match.single_input_node(0)
conf_node = match.single_input_node(1)
prior_boxes_node = match.single_input_node(2)
locs_out_nodes = locs_node[0].out_nodes()
assert len(locs_out_nodes) == 1
locs_out_node = locs_out_nodes[list(locs_out_nodes.keys())[0]]
assert locs_out_node.op == "OpOutput", locs_out_node.op
graph.remove_node(locs_out_node.id)
conf_out_nodes = conf_node[0].out_nodes()
assert len(conf_out_nodes) == 1
conf_out_node = conf_out_nodes[list(conf_out_nodes.keys())[0]]
assert conf_out_node.op == "OpOutput", conf_out_node.op
graph.remove_node(conf_out_node.id)
# reshape operation to flatten confidence tensor
reshape_loc_op = Reshape(graph, {'dim': np.array([0, -1])})
reshape_loc_node = reshape_loc_op.create_node(
[locs_node], dict(name='DetectionOutput_Reshape_loc_'))
# reshape operation to flatten confidence tensor
reshape_conf_op = Reshape(graph, {'dim': np.array([0, -1])})
reshape_conf_node = reshape_conf_op.create_node(
[conf_node], dict(name='DetectionOutput_Reshape_conf_'))
# remove the OpOutput node after the priors node
assert prior_boxes_node[0].out_node().op == "OpOutput"
graph.remove_node(prior_boxes_node[0].out_node().id)
# reshape operation for prior boxes tensor
reshape_priors_op = Reshape(graph, {'dim': np.array([1, 2, -1])})
reshape_priors_node = reshape_priors_op.create_node(
[prior_boxes_node], dict(name='DetectionOutput_Reshape_priors_'))
# create Detection Output node with three inputs: locations, confidences and prior boxes
detection_output_op = DetectionOutput(
graph, match.custom_replacement_desc.custom_attributes)
detection_output_node = detection_output_op.create_node(
[reshape_loc_node, reshape_conf_node, reshape_priors_node],
dict(name=detection_output_op.attrs['type'] + '_'))
PermuteAttrs.set_permutation(reshape_priors_node,
detection_output_node, None)
# create Output node to mark DetectionOutput as a graph output operation
output_op = Output(graph)
output_op.create_node([detection_output_node], dict(name='sink_'))
return {}
def replace_sub_graph(self, graph: nx.MultiDiGraph, match: dict):
"""
Need to find the pattern: SoftmaxActivation -> DetectionOutput
DetectionOutput in IE expects flattened input from SoftMax, that is why there is the need to add
Flatten layer
Parameters
----------
graph : nx.MultiDiGraph
Graph with loaded model.
match : dict
Patterns which were found in graph structure.
"""
softmax_activation = match['softmax_activation']
multi_box_detection = match['multi_box_detection']
softmax_activation['axis'] = -1
edge_data = graph.get_edge_data(softmax_activation.id, multi_box_detection.id)
out_port = edge_data[0]['out']
in_port = edge_data[0]['in']
graph.remove_edge(softmax_activation.id, multi_box_detection.id)
symbol_node = dict(
op='Flatten',
name=multi_box_detection.name + '/Reshape_',
dim=[0, -1],
axis=1,
end_axis=-1
)
new_reshape_op = Reshape(graph, {'symbol_dict': symbol_node})
new_reshape_node = new_reshape_op.create_node([softmax_activation])
new_reshape_node['dim'] = [0, -1]
create_edge(new_reshape_node, multi_box_detection, in_port=in_port, out_port=out_port)
def replace_op(self, graph: nx.MultiDiGraph, node: Node):
# reshape tensor with batch indices to 2d
unsqueeze_op = Unsqueeze(
graph, {'unsqueeze_dims': np.array([1], dtype=np.int64)})
unsqueeze_node = unsqueeze_op.create_node([node.in_node(2)])
concat_op = Concat(
graph, {
'axis': 1,
'name': node.name + '/concat_batch_indices_and_boxes'
})
concat_node = concat_op.create_node([unsqueeze_node, node.in_node(1)])
# do not remove edge with crop_size because it is needed in the partial infer
graph.remove_edge(node.in_node(1).id, node.id)
# input to the CropAndResize contains boxes coordinates in YXYX layout. But IE layer ROIPooling expects
# coordinates in the XYXY layout, so convolution is added here to swap coordinates
swapped_box_coordinates_node = add_convolution_to_swap_xy_coordinates(
graph, concat_node, 5)
# reshape locations tensor to 2D so it could be passed to Eltwise which will be converted to ScaleShift
reshape_2d_op = Reshape(graph, dict(dim=np.array([-1, 5])))
reshape_2d_node = reshape_2d_op.create_node(
[swapped_box_coordinates_node], dict(name='reshape_2d_'))
create_edge(reshape_2d_node, node, 0, 1)
# do not replace any output edge
return []
def replace_pattern(self, graph: Graph, match: dict):
if match['axis'].value is None or match['input'].shape is None:
return
dims = len(match['input'].shape)
ones = np.ones(dims, dtype=np.int64)
axis = np.array(match['axis'].value)
axis = axis if axis.ndim != 0 else np.array([axis], dtype=np.int64)
mean = graph.node[match['mean'].node]
mean['stride'] = np.array(ones)
# TODO: need to check axis with real layout
spatial_dims = np.array(axis)
mean['spatial_dims'] = spatial_dims
mean['pad'] = np.zeros((dims, 2), np.int64)
mean['pad_spatial_shape'] = np.array(mean['pad'][spatial_dims])
window = np.array(ones)
window[spatial_dims] = match['input'].shape[spatial_dims]
mean['window'] = window
mean['TF_op'] = mean['op']
mean['op'] = 'AvgPool'
mean['pool_method'] = 'avg'
mean['rounding_type'] = 'ceil'
mean['exclude_pad'] = 'true'
mean['kernel_spatial'] = window[spatial_dims]
graph.remove_edge(match['axis'].node, match['mean'].node)
mean['permute_attrs'] = PermuteAttrs().update_attrs(attrs=[(
'pad',
'input:0'), ('stride',
'input:0'), ('window',
'input:0'), ('spatial_dims', 'input:0')])
if match['mean'].keep_dims == False:
output = match['mean'].out_node()
pool_node = match['mean']
# Keep dims for AvgPool
shape = np.array(output.shape)
for idx in spatial_dims:
shape = np.insert(shape, idx, 1)
graph.remove_edge(pool_node.id, output.id)
# Create new data for pool with all dims
pool_data = Op.create_data_node(graph, pool_node,
{'shape': np.array(shape)})
# Create and connect reshape node
reshape_op = Reshape(graph, {'dim': np.array(output.shape)})
reshape_node = reshape_op.create_node(
[pool_data],
dict(name='Reshape_',
permute_attrs=PermuteAttrs().update_attrs(
attrs=[('dim', 'output:0')])))
graph.create_edge(reshape_node, output)
def add_convolution_to_swap_xy_coordinates(graph: nx.MultiDiGraph,
input_node: Node,
coordinates_size: int):
"""
The function add convolution node after the node 'input_node' to swap xy coordinates of the boxes produced
by the node 'input_node'. It is expected that box coordinates are located in the fastest changing dimension of the
'input_node' output, i.e. the input tensor could be reshaped to [num_boxes, 4] or [num_boxes, 5]. If the size is 5,
then the 0-th element for each of num_boxes blocks is not changed and element 1 is swapped with element 2, element 3
is swapped with element 4. This is the case when boxes coordinates are produced by the layer "Proposal". The exact
amount of elements in each block is equal to the 'coordinates_size' parameter.
:param graph: graph to operate on.
:param input_node: node producing boxes coordinates.
:param coordinates_size: integer value equal to 4 or 5.
:return convolution node that swaps coordinates.
"""
# swap of input tensor with 4 or 5 numbers describing boxes are supported
assert (coordinates_size in [4, 5])
input_reshape_4d_op = Reshape(
input_node.graph, dict(dim=np.array([-1, 1, 1, coordinates_size])))
input_reshape_4d_node = input_reshape_4d_op.create_node(
[input_node], dict(name=input_node.name + '/reshape_4d'))
update_attrs(input_reshape_4d_node, 'shape_attrs', 'dim')
if coordinates_size == 5:
# zero indexed element is not box coordinate ("batch id" in case of Proposal)
conv_filter_data = np.array(
np.array([[[[1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0],
[0, 0, 0, 0, 1], [0, 0, 0, 1, 0]]]],
dtype=np.float32))
else:
conv_filter_data = np.array(
np.array(
[[[[0, 1, 0, 0], [1, 0, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0]]]],
dtype=np.float32))
conv_filter_const_op = Const(graph, dict(value=conv_filter_data))
conv_filter_const_node = conv_filter_const_op.create_node(
[], dict(name=input_node.name + '/weights'))
conv_op = Convolution(
graph, {
'bias_addable': True,
'channel_dims': np.array([3]),
'batch_dims': np.array([0]),
'input_feature_channel': 2,
'output_feature_channel': 3,
'group': 1,
'layout': 'NHWC',
})
return conv_op.create_node([input_reshape_4d_node, conv_filter_const_node],
dict(name=input_node.name + "/conv"))
def replace_op(self, graph: Graph, node: Node):
if node.has_and_set('inputs_preprocessed'):
log.debug('Node "{}" has already been preprocessed'.format(
node.soft_get('name')))
return []
# reshape tensor with batch indices to 2d
unsqueeze_op = Unsqueeze(
graph, {'unsqueeze_dims': np.array([1], dtype=np.int64)})
unsqueeze_node = unsqueeze_op.create_node([node.in_node(2)])
concat_op = Concat(
graph, {
'axis': 1,
'name': node.name + '/concat_batch_indices_and_boxes',
'in_ports_count': 2
})
concat_node = concat_op.create_node([unsqueeze_node, node.in_node(1)])
# do not remove edge with crop_size because it is needed in the partial infer
graph.remove_edge(node.in_node(1).id, node.id)
# input to the CropAndResize contains boxes coordinates in YXYX layout. But IE layer ROIPooling expects
# coordinates in the XYXY layout, so convolution is added here to swap coordinates
swapped_box_coordinates_node = add_convolution_to_swap_xy_coordinates(
graph, concat_node, 5)
# reshape locations tensor to 2D so it could be passed to Eltwise which will be converted to ScaleShift
reshape_2d_op = Reshape(graph, dict(dim=np.array([-1, 5])))
reshape_2d_node = reshape_2d_op.create_node(
[swapped_box_coordinates_node],
dict(name=swapped_box_coordinates_node.id + '/reshape_2d_',
nchw_layout=True))
graph.create_edge(reshape_2d_node, node, 0, 1)
# do not replace any output edge
return []
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch):
reshape_classes_op = Reshape(graph, {'dim': np.array([0, -1])})
reshape_classes_node = reshape_classes_op.create_node(
[match.single_input_node(1)[0]], dict(name='do_reshape_classes'))
priors_node = match.single_input_node(2)[0]
placeholder = [
Node(graph, node_id) for node_id in graph.nodes()
if Node(graph, node_id).op == 'Placeholder'
][0]
im_height = placeholder.shape[1]
im_width = placeholder.shape[2]
# scale prior boxes to the [0, 1] interval
priors_scale_const_node = Const(
graph, {
'value':
np.array(
[1 / im_width, 1 / im_height, 1 / im_width, 1 / im_height])
}).create_node([])
priors_scale_node = Eltwise(graph, {
'name': 'scale_priors',
'operation': 'mul'
}).create_node([priors_node, priors_scale_const_node])
# calculate prior boxes widths and heights
split_node = SplitV(graph, {
'axis': 2,
'size_splits': [1, 1, 1, 1],
'out_ports_count': 4
}).create_node([priors_scale_node])
priors_width_node = __class__._create_sub(graph, split_node, 2,
split_node, 0)
priors_height_node = __class__._create_sub(graph, split_node, 3,
split_node, 1)
# concat weights and heights into a single tensor and multiple with the box coordinates regression values
concat_width_height_node = Concat(graph, {
'name': 'concat_priors_width_height',
'axis': -1,
'in_ports_count': 4
}).create_node([
priors_width_node, priors_height_node, priors_width_node,
priors_height_node
])
applied_width_height_regressions_node = Eltwise(graph, {'name': 'final_regressions', 'operation': 'mul'}). \
create_node([concat_width_height_node, match.single_input_node(0)[0]])
# reshape to 2D tensor as Inference Engine Detection Output layer expects
reshape_regression_op = Reshape(graph, {'dim': np.array([0, -1])})
reshape_regression_node = reshape_regression_op.create_node(
[applied_width_height_regressions_node],
{'name': 'reshape_regression'})
detection_output_op = DetectionOutput(
graph, match.custom_replacement_desc.custom_attributes)
detection_output_op.attrs['old_infer'] = detection_output_op.attrs[
'infer']
detection_output_op.attrs['infer'] = __class__.do_infer
detection_output_node = detection_output_op.create_node(
[reshape_regression_node, reshape_classes_node, priors_scale_node],
dict(name=detection_output_op.attrs['type'],
clip=1,
normalized=1,
variance_encoded_in_target=0))
return {'detection_output_node': detection_output_node}