def generate_sub_graph(self, graph: Graph, match: SubgraphMatch):
# IE DetectionOutput layer consumes flattened confidences and locations tensors.
# That is why we add reshapes before them.
locs_node = match.single_input_node(0)
conf_node = match.single_input_node(1)
prior_boxes_node = match.single_input_node(2)
locs_out_nodes = locs_node[0].out_nodes()
assert len(locs_out_nodes) == 1
locs_out_node = locs_out_nodes[list(locs_out_nodes.keys())[0]]
assert locs_out_node.op == "Result", locs_out_node.op
graph.remove_node(locs_out_node.id)
conf_out_nodes = conf_node[0].out_nodes()
assert len(conf_out_nodes) == 1
conf_out_node = conf_out_nodes[list(conf_out_nodes.keys())[0]]
assert conf_out_node.op == "Result", conf_out_node.op
graph.remove_node(conf_out_node.id)
# reshape operation to flatten confidence tensor
const = Const(graph, {'value': int64_array([0, -1])}).create_node()
reshape_loc_node = Reshape(graph, {}).create_node(
[locs_node, const], dict(name='DetectionOutput_Reshape_loc_'))
# reshape operation to flatten confidence tensor
reshape_conf_node = Reshape(graph, {}).create_node(
[conf_node, const], dict(name='DetectionOutput_Reshape_conf_'))
# remove the Result node after the priors node
assert prior_boxes_node[0].out_node().op == "Result"
graph.remove_node(prior_boxes_node[0].out_node().id)
# reshape operation for prior boxes tensor
const = Const(graph, {'value': int64_array([1, 2, -1])}).create_node()
reshape_priors_node = Reshape(graph, {}).create_node(
[prior_boxes_node, const],
dict(name='DetectionOutput_Reshape_priors_'))
# create Detection Output node with three inputs: locations, confidences and prior boxes
detection_output_op = DetectionOutput(
graph, match.custom_replacement_desc.custom_attributes)
detection_output_node = detection_output_op.create_node(
[reshape_loc_node, reshape_conf_node, reshape_priors_node],
dict(name=detection_output_op.attrs['type'] + '_'))
PermuteAttrs.set_permutation(reshape_priors_node,
detection_output_node, None)
# create Output node to mark DetectionOutput as a graph output operation
output_op = Result(graph)
output_op.create_node([detection_output_node], dict(name='sink_'))
return {}
def nodes_to_remove(self, graph: Graph, match: SubgraphMatch):
new_nodes_to_remove = match.matched_nodes_names()
new_nodes_to_remove.remove(match.single_input_node(0)[0].id)
new_nodes_to_remove.remove(match.single_input_node(1)[0].id)
new_nodes_to_remove.remove(match.single_input_node(2)[0].id)
return new_nodes_to_remove
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch):
reshape_classes_node = create_op_node_with_second_input(graph, Reshape, int64_array([0, -1]),
dict(name='do_reshape_classes'),
match.single_input_node(1)[0])
initial_priors_node = match.single_input_node(2)[0]
priors_name = initial_priors_node.soft_get('name', initial_priors_node.id)
# model calculates identical prior boxes for each batch, so we take first slice of them
begin = Const(graph, {'value': mo_array([0, 0, 0], dtype=np.int32)}).create_node()
end = Const(graph, {'value': mo_array([1, 0, 0], dtype=np.int32)}).create_node()
stride = Const(graph, {'value': mo_array([1, 1, 1], dtype=np.int32)}).create_node()
priors_node = StridedSlice(graph, {'name': priors_name + '/0_batch_slice',
'begin_mask': int64_array([1, 1, 1]),
'end_mask': int64_array([1, 0, 0]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])}).create_node()
initial_priors_node.out_port(0).connect(priors_node.in_port(0))
begin.out_port(0).connect(priors_node.in_port(1))
end.out_port(0).connect(priors_node.in_port(2))
stride.out_port(0).connect(priors_node.in_port(3))
placeholders = graph.get_op_nodes(type='Parameter')
assert len(placeholders) == 1, "{} replacer requires model to have one Placeholder, but current model has " \
"{} placeholders".format(self.replacement_id, len(placeholders))
placeholder = placeholders[0]
# scale prior boxes to the [0, 1] interval
node_with_scales_for_prior_boxes = self.placeholder_scales(placeholder)
priors_scale_node = Mul(graph, {'name': 'scale_priors'}).create_node()
broadcast = Broadcast(graph, {'name': 'scales_broadcast'}).create_node()
shape_of_priors = Shape(graph, {'name': 'priors_shape'}).create_node()
priors_node.out_port(0).connect(shape_of_priors.in_port(0))
broadcast.in_port(1).connect(shape_of_priors.out_port(0))
broadcast.in_port(0).connect(node_with_scales_for_prior_boxes.out_port(0))
priors_scale_node.in_port(0).connect(priors_node.out_port(0))
priors_scale_node.in_port(1).connect(broadcast.out_port(0))
try:
variance = match.custom_replacement_desc.custom_attributes['variance']
except:
raise Error('There is no variance attribute in {} replacement config file `custom_attributes`'
''.format(self.replacement_id))
priors = self.append_variances(priors_scale_node, variance)
# calculate prior boxes widths and heights
split_node = create_op_with_const_inputs(
graph, VariadicSplit, {1: int64_array(2), 2: int64_array([1, 1, 1, 1])}, {'out_ports_count': 4},
priors_scale_node)
priors_width_node = Sub(graph, dict(name=split_node.name + '/sub_2-0_')
).create_node([(split_node, 2), (split_node, 0)])
priors_height_node = Sub(graph, dict(name=split_node.name + '/sub_3-1_')
).create_node([(split_node, 3), (split_node, 1)])
# concat weights and heights into a single tensor and multiple with the box coordinates regression values
# WA with 3 Concats instead of 1 for keeping model reshapable
# concat_width_height_node = Concat(graph, {'name': 'concat_priors_width_height', 'axis': -1,
# 'in_ports_count': 4}).create_node(
# [priors_width_node, priors_height_node, priors_width_node, priors_height_node])
concat_1 = Concat(graph, {'name': 'concat_width_height',
'axis': -1, 'in_ports_count': 2}).create_node([priors_width_node, priors_height_node])
concat_2 = Concat(graph, {'name': 'concat_width_height_width',
'axis': -1, 'in_ports_count': 2}).create_node([concat_1, priors_width_node])
concat_width_height_node = Concat(graph, {'name': 'concat_priors_width_height', 'axis': -1, 'in_ports_count': 2}
).create_node([concat_2, priors_height_node])
applied_width_height_regressions_node = Mul(graph, {'name': 'final_regressions'}).create_node(
[concat_width_height_node, match.single_input_node(0)[0]])
# reshape to 2D tensor as Inference Engine Detection Output layer expects
reshape_regression_node = create_op_node_with_second_input(graph, Reshape, int64_array([0, -1]),
dict(name='reshape_regression'),
applied_width_height_regressions_node)
detection_output_op = DetectionOutput(graph, match.custom_replacement_desc.custom_attributes)
# get nms from the original network
iou_threshold = None
nms_nodes = graph.get_op_nodes(op='NonMaxSuppression')
if len(nms_nodes) > 0:
# it is highly unlikely that for different classes NMS has different
# moreover DetectionOutput accepts only scalar values for iou_threshold (nms_threshold)
iou_threshold = nms_nodes[0].in_node(3).value
if iou_threshold is None:
raise Error('During {} `iou_threshold` was not retrieved from RetinaNet graph'.format(self.replacement_id))
detection_output_node = detection_output_op.create_node(
[reshape_regression_node, reshape_classes_node, priors],
dict(name=detection_output_op.attrs['type'], nms_threshold=iou_threshold, clip_after_nms=1, normalized=1,
variance_encoded_in_target=0, background_label_id=1000))
# As outputs are replaced with a postprocessing node, outgoing tensor names are no longer
# correspond to original tensors and should be removed from output->Result edges
out_nodes = []
for out in range(match.outputs_count()):
out_nodes.append(match.output_node(out)[0])
clear_tensor_names_info(out_nodes)
return {'detection_output_node': detection_output_node}