def generate_sub_graph(self, graph: Graph, match: SubgraphMatch): # IE DetectionOutput layer consumes flattened confidences and locations tensors. # That is why we add reshapes before them. locs_node = match.single_input_node(0) conf_node = match.single_input_node(1) prior_boxes_node = match.single_input_node(2) locs_out_nodes = locs_node[0].out_nodes() assert len(locs_out_nodes) == 1 locs_out_node = locs_out_nodes[list(locs_out_nodes.keys())[0]] assert locs_out_node.op == "OpOutput", locs_out_node.op graph.remove_node(locs_out_node.id) conf_out_nodes = conf_node[0].out_nodes() assert len(conf_out_nodes) == 1 conf_out_node = conf_out_nodes[list(conf_out_nodes.keys())[0]] assert conf_out_node.op == "OpOutput", conf_out_node.op graph.remove_node(conf_out_node.id) # reshape operation to flatten confidence tensor reshape_loc_op = Reshape(graph, {'dim': np.array([0, -1])}) reshape_loc_node = reshape_loc_op.create_node( [locs_node], dict(name='DetectionOutput_Reshape_loc_')) # reshape operation to flatten confidence tensor reshape_conf_op = Reshape(graph, {'dim': np.array([0, -1])}) reshape_conf_node = reshape_conf_op.create_node( [conf_node], dict(name='DetectionOutput_Reshape_conf_')) # remove the OpOutput node after the priors node assert prior_boxes_node[0].out_node().op == "OpOutput" graph.remove_node(prior_boxes_node[0].out_node().id) # reshape operation for prior boxes tensor reshape_priors_op = Reshape(graph, {'dim': np.array([1, 2, -1])}) reshape_priors_node = reshape_priors_op.create_node( [prior_boxes_node], dict(name='DetectionOutput_Reshape_priors_')) # create Detection Output node with three inputs: locations, confidences and prior boxes detection_output_op = DetectionOutput( graph, match.custom_replacement_desc.custom_attributes) detection_output_node = detection_output_op.create_node( [reshape_loc_node, reshape_conf_node, reshape_priors_node], dict(name=detection_output_op.attrs['type'] + '_')) PermuteAttrs.set_permutation(reshape_priors_node, detection_output_node, None) # create Output node to mark DetectionOutput as a graph output operation output_op = Output(graph) output_op.create_node([detection_output_node], dict(name='sink_')) return {}
def nodes_to_remove(self, graph: Graph, match: SubgraphMatch): new_nodes_to_remove = match.matched_nodes_names() new_nodes_to_remove.remove(match.single_input_node(0)[0].id) new_nodes_to_remove.remove(match.single_input_node(1)[0].id) new_nodes_to_remove.remove(match.single_input_node(2)[0].id) return new_nodes_to_remove
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch): reshape_classes_node = create_op_node_with_second_input( graph, Reshape, int64_array([0, -1]), dict(name='do_reshape_classes'), match.single_input_node(1)[0]) priors_node = match.single_input_node(2)[0] placeholder = [ Node(graph, node_id) for node_id in graph.nodes() if Node(graph, node_id).op == 'Parameter' ][0] im_height = placeholder.shape[1] im_width = placeholder.shape[2] # scale prior boxes to the [0, 1] interval priors_scale_const_node = Const( graph, { 'value': np.array( [1 / im_width, 1 / im_height, 1 / im_width, 1 / im_height]) }).create_node([]) priors_scale_node = Mul(graph, { 'name': 'scale_priors' }).create_node([priors_node, priors_scale_const_node]) # calculate prior boxes widths and heights split_node = SplitV(graph, { 'axis': 2, 'size_splits': [1, 1, 1, 1], 'out_ports_count': 4 }).create_node([priors_scale_node]) priors_width_node = Sub( graph, dict(name=split_node.name + '/sub_2-0_')).create_node([ (split_node, 2), (split_node, 0) ]) priors_height_node = Sub(graph, dict(name=split_node.name + '/sub_3-1_')).create_node([ (split_node, 3), (split_node, 1) ]) # concat weights and heights into a single tensor and multiple with the box coordinates regression values concat_width_height_node = Concat(graph, { 'name': 'concat_priors_width_height', 'axis': -1, 'in_ports_count': 4 }).create_node([ priors_width_node, priors_height_node, priors_width_node, priors_height_node ]) applied_width_height_regressions_node = Mul(graph, { 'name': 'final_regressions' }).create_node( [concat_width_height_node, match.single_input_node(0)[0]]) # reshape to 2D tensor as Inference Engine Detection Output layer expects reshape_regression_node = create_op_node_with_second_input( graph, Reshape, int64_array([0, -1]), dict(name='reshape_regression'), applied_width_height_regressions_node) detection_output_op = DetectionOutput( graph, match.custom_replacement_desc.custom_attributes) detection_output_op.attrs['old_infer'] = detection_output_op.attrs[ 'infer'] detection_output_op.attrs['infer'] = __class__.do_infer detection_output_node = detection_output_op.create_node( [reshape_regression_node, reshape_classes_node, priors_scale_node], dict(name=detection_output_op.attrs['type'], clip=1, normalized=1, variance_encoded_in_target=0)) return {'detection_output_node': detection_output_node}
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch): reshape_classes_node = create_op_node_with_second_input( graph, Reshape, int64_array([0, -1]), dict(name='do_reshape_classes'), match.single_input_node(1)[0]) initial_priors_node = match.single_input_node(2)[0] priors_name = initial_priors_node.soft_get('name', initial_priors_node.id) # model calculates identical prior boxes for each batch, so we take first slice of them begin = Const(graph, { 'value': np.array([0, 0, 0], dtype=np.int32) }).create_node() end = Const(graph, { 'value': np.array([1, 0, 0], dtype=np.int32) }).create_node() stride = Const(graph, { 'value': np.array([1, 1, 1], dtype=np.int32) }).create_node() priors_node = StridedSlice( graph, { 'name': priors_name + '/0_batch_slice', 'begin_mask': np.array([1, 1, 1], dtype=np.int32), 'end_mask': np.array([1, 0, 0], dtype=np.int32), 'new_axis_mask': np.array([0], dtype=np.int32), 'shrink_axis_mask': np.array([0], dtype=np.int32), 'ellipsis_mask': np.array([0], dtype=np.int32) }).create_node() initial_priors_node.out_port(0).connect(priors_node.in_port(0)) begin.out_port(0).connect(priors_node.in_port(1)) end.out_port(0).connect(priors_node.in_port(2)) stride.out_port(0).connect(priors_node.in_port(3)) placeholders = graph.get_op_nodes(type='Parameter') assert len(placeholders) == 1, "{} replacer requires model to have one Placeholder, but current model has " \ "{} placeholders".format(self.replacement_id, len(placeholders)) placeholder = placeholders[0] # scale prior boxes to the [0, 1] interval node_with_scales_for_prior_boxes = self.placeholder_scales(placeholder) priors_scale_node = Mul(graph, {'name': 'scale_priors'}).create_node() broadcast = Broadcast(graph, { 'name': 'scales_broadcast' }).create_node() shape_of_priors = Shape(graph, {'name': 'priors_shape'}).create_node() priors_node.out_port(0).connect(shape_of_priors.in_port(0)) broadcast.in_port(1).connect(shape_of_priors.out_port(0)) broadcast.in_port(0).connect( node_with_scales_for_prior_boxes.out_port(0)) priors_scale_node.in_port(0).connect(priors_node.out_port(0)) priors_scale_node.in_port(1).connect(broadcast.out_port(0)) try: variance = match.custom_replacement_desc.custom_attributes[ 'variance'] except: raise Error( 'There is no variance attribute in {} replacement config file `custom_attributes`' ''.format(self.replacement_id)) priors = self.append_variances(priors_scale_node, variance) # calculate prior boxes widths and heights split_node = create_op_with_const_inputs(graph, VariadicSplit, { 1: int64_array(2), 2: int64_array([1, 1, 1, 1]) }, {'out_ports_count': 4}, priors_scale_node) priors_width_node = Sub( graph, dict(name=split_node.name + '/sub_2-0_')).create_node([ (split_node, 2), (split_node, 0) ]) priors_height_node = Sub(graph, dict(name=split_node.name + '/sub_3-1_')).create_node([ (split_node, 3), (split_node, 1) ]) # concat weights and heights into a single tensor and multiple with the box coordinates regression values # WA with 3 Concats instead of 1 for keeping model reshapable # concat_width_height_node = Concat(graph, {'name': 'concat_priors_width_height', 'axis': -1, # 'in_ports_count': 4}).create_node( # [priors_width_node, priors_height_node, priors_width_node, priors_height_node]) concat_1 = Concat(graph, { 'name': 'concat_width_height', 'axis': -1, 'in_ports_count': 2 }).create_node([priors_width_node, priors_height_node]) concat_2 = Concat(graph, { 'name': 'concat_width_height_width', 'axis': -1, 'in_ports_count': 2 }).create_node([concat_1, priors_width_node]) concat_width_height_node = Concat(graph, { 'name': 'concat_priors_width_height', 'axis': -1, 'in_ports_count': 2 }).create_node([concat_2, priors_height_node]) applied_width_height_regressions_node = Mul(graph, { 'name': 'final_regressions' }).create_node( [concat_width_height_node, match.single_input_node(0)[0]]) # reshape to 2D tensor as Inference Engine Detection Output layer expects reshape_regression_node = create_op_node_with_second_input( graph, Reshape, int64_array([0, -1]), dict(name='reshape_regression'), applied_width_height_regressions_node) detection_output_op = DetectionOutput( graph, match.custom_replacement_desc.custom_attributes) # get nms from the original network iou_threshold = None nms_nodes = graph.get_op_nodes(op='NonMaxSuppression') if len(nms_nodes) > 0: # it is highly unlikely that for different classes NMS has different # moreover DetectionOutput accepts only scalar values for iou_threshold (nms_threshold) iou_threshold = nms_nodes[0].in_node(3).value if iou_threshold is None: raise Error( 'During {} `iou_threshold` was not retrieved from RetinaNet graph' .format(self.replacement_id)) detection_output_node = detection_output_op.create_node( [reshape_regression_node, reshape_classes_node, priors], dict(name=detection_output_op.attrs['type'], nms_threshold=iou_threshold, clip_after_nms=1, normalized=1, variance_encoded_in_target=0, background_label_id=1000)) return {'detection_output_node': detection_output_node}