def generate_sub_graph(self, graph: Graph, match: SubgraphMatch): # IE DetectionOutput layer consumes flattened confidences and locations tensors. # That is why we add reshapes before them. locs_node = match.single_input_node(0) conf_node = match.single_input_node(1) prior_boxes_node = match.single_input_node(2) locs_out_nodes = locs_node[0].out_nodes() assert len(locs_out_nodes) == 1 locs_out_node = locs_out_nodes[list(locs_out_nodes.keys())[0]] assert locs_out_node.op == "OpOutput", locs_out_node.op graph.remove_node(locs_out_node.id) conf_out_nodes = conf_node[0].out_nodes() assert len(conf_out_nodes) == 1 conf_out_node = conf_out_nodes[list(conf_out_nodes.keys())[0]] assert conf_out_node.op == "OpOutput", conf_out_node.op graph.remove_node(conf_out_node.id) # reshape operation to flatten confidence tensor reshape_loc_op = Reshape(graph, {'dim': np.array([0, -1])}) reshape_loc_node = reshape_loc_op.create_node( [locs_node], dict(name='DetectionOutput_Reshape_loc_')) # reshape operation to flatten confidence tensor reshape_conf_op = Reshape(graph, {'dim': np.array([0, -1])}) reshape_conf_node = reshape_conf_op.create_node( [conf_node], dict(name='DetectionOutput_Reshape_conf_')) # remove the OpOutput node after the priors node assert prior_boxes_node[0].out_node().op == "OpOutput" graph.remove_node(prior_boxes_node[0].out_node().id) # reshape operation for prior boxes tensor reshape_priors_op = Reshape(graph, {'dim': np.array([1, 2, -1])}) reshape_priors_node = reshape_priors_op.create_node( [prior_boxes_node], dict(name='DetectionOutput_Reshape_priors_')) # create Detection Output node with three inputs: locations, confidences and prior boxes detection_output_op = DetectionOutput( graph, match.custom_replacement_desc.custom_attributes) detection_output_node = detection_output_op.create_node( [reshape_loc_node, reshape_conf_node, reshape_priors_node], dict(name=detection_output_op.attrs['type'] + '_')) PermuteAttrs.set_permutation(reshape_priors_node, detection_output_node, None) # create Output node to mark DetectionOutput as a graph output operation output_op = Output(graph) output_op.create_node([detection_output_node], dict(name='sink_')) return {}
def replace_sub_graph(self, graph: nx.MultiDiGraph, match: dict): """ Need to find the pattern: SoftmaxActivation -> DetectionOutput DetectionOutput in IE expects flattened input from SoftMax, that is why there is the need to add Flatten layer Parameters ---------- graph : nx.MultiDiGraph Graph with loaded model. match : dict Patterns which were found in graph structure. """ softmax_activation = match['softmax_activation'] multi_box_detection = match['multi_box_detection'] softmax_activation['axis'] = -1 edge_data = graph.get_edge_data(softmax_activation.id, multi_box_detection.id) out_port = edge_data[0]['out'] in_port = edge_data[0]['in'] graph.remove_edge(softmax_activation.id, multi_box_detection.id) symbol_node = dict( op='Flatten', name=multi_box_detection.name + '/Reshape_', dim=[0, -1], axis=1, end_axis=-1 ) new_reshape_op = Reshape(graph, {'symbol_dict': symbol_node}) new_reshape_node = new_reshape_op.create_node([softmax_activation]) new_reshape_node['dim'] = [0, -1] create_edge(new_reshape_node, multi_box_detection, in_port=in_port, out_port=out_port)
def replace_op(self, graph: nx.MultiDiGraph, node: Node): # reshape tensor with batch indices to 2d unsqueeze_op = Unsqueeze( graph, {'unsqueeze_dims': np.array([1], dtype=np.int64)}) unsqueeze_node = unsqueeze_op.create_node([node.in_node(2)]) concat_op = Concat( graph, { 'axis': 1, 'name': node.name + '/concat_batch_indices_and_boxes' }) concat_node = concat_op.create_node([unsqueeze_node, node.in_node(1)]) # do not remove edge with crop_size because it is needed in the partial infer graph.remove_edge(node.in_node(1).id, node.id) # input to the CropAndResize contains boxes coordinates in YXYX layout. But IE layer ROIPooling expects # coordinates in the XYXY layout, so convolution is added here to swap coordinates swapped_box_coordinates_node = add_convolution_to_swap_xy_coordinates( graph, concat_node, 5) # reshape locations tensor to 2D so it could be passed to Eltwise which will be converted to ScaleShift reshape_2d_op = Reshape(graph, dict(dim=np.array([-1, 5]))) reshape_2d_node = reshape_2d_op.create_node( [swapped_box_coordinates_node], dict(name='reshape_2d_')) create_edge(reshape_2d_node, node, 0, 1) # do not replace any output edge return []
def replace_pattern(self, graph: Graph, match: dict): if match['axis'].value is None or match['input'].shape is None: return dims = len(match['input'].shape) ones = np.ones(dims, dtype=np.int64) axis = np.array(match['axis'].value) axis = axis if axis.ndim != 0 else np.array([axis], dtype=np.int64) mean = graph.node[match['mean'].node] mean['stride'] = np.array(ones) # TODO: need to check axis with real layout spatial_dims = np.array(axis) mean['spatial_dims'] = spatial_dims mean['pad'] = np.zeros((dims, 2), np.int64) mean['pad_spatial_shape'] = np.array(mean['pad'][spatial_dims]) window = np.array(ones) window[spatial_dims] = match['input'].shape[spatial_dims] mean['window'] = window mean['TF_op'] = mean['op'] mean['op'] = 'AvgPool' mean['pool_method'] = 'avg' mean['rounding_type'] = 'ceil' mean['exclude_pad'] = 'true' mean['kernel_spatial'] = window[spatial_dims] graph.remove_edge(match['axis'].node, match['mean'].node) mean['permute_attrs'] = PermuteAttrs().update_attrs(attrs=[( 'pad', 'input:0'), ('stride', 'input:0'), ('window', 'input:0'), ('spatial_dims', 'input:0')]) if match['mean'].keep_dims == False: output = match['mean'].out_node() pool_node = match['mean'] # Keep dims for AvgPool shape = np.array(output.shape) for idx in spatial_dims: shape = np.insert(shape, idx, 1) graph.remove_edge(pool_node.id, output.id) # Create new data for pool with all dims pool_data = Op.create_data_node(graph, pool_node, {'shape': np.array(shape)}) # Create and connect reshape node reshape_op = Reshape(graph, {'dim': np.array(output.shape)}) reshape_node = reshape_op.create_node( [pool_data], dict(name='Reshape_', permute_attrs=PermuteAttrs().update_attrs( attrs=[('dim', 'output:0')]))) graph.create_edge(reshape_node, output)
def add_convolution_to_swap_xy_coordinates(graph: nx.MultiDiGraph, input_node: Node, coordinates_size: int): """ The function add convolution node after the node 'input_node' to swap xy coordinates of the boxes produced by the node 'input_node'. It is expected that box coordinates are located in the fastest changing dimension of the 'input_node' output, i.e. the input tensor could be reshaped to [num_boxes, 4] or [num_boxes, 5]. If the size is 5, then the 0-th element for each of num_boxes blocks is not changed and element 1 is swapped with element 2, element 3 is swapped with element 4. This is the case when boxes coordinates are produced by the layer "Proposal". The exact amount of elements in each block is equal to the 'coordinates_size' parameter. :param graph: graph to operate on. :param input_node: node producing boxes coordinates. :param coordinates_size: integer value equal to 4 or 5. :return convolution node that swaps coordinates. """ # swap of input tensor with 4 or 5 numbers describing boxes are supported assert (coordinates_size in [4, 5]) input_reshape_4d_op = Reshape( input_node.graph, dict(dim=np.array([-1, 1, 1, coordinates_size]))) input_reshape_4d_node = input_reshape_4d_op.create_node( [input_node], dict(name=input_node.name + '/reshape_4d')) update_attrs(input_reshape_4d_node, 'shape_attrs', 'dim') if coordinates_size == 5: # zero indexed element is not box coordinate ("batch id" in case of Proposal) conv_filter_data = np.array( np.array([[[[1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 0, 0, 0, 1], [0, 0, 0, 1, 0]]]], dtype=np.float32)) else: conv_filter_data = np.array( np.array( [[[[0, 1, 0, 0], [1, 0, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0]]]], dtype=np.float32)) conv_filter_const_op = Const(graph, dict(value=conv_filter_data)) conv_filter_const_node = conv_filter_const_op.create_node( [], dict(name=input_node.name + '/weights')) conv_op = Convolution( graph, { 'bias_addable': True, 'channel_dims': np.array([3]), 'batch_dims': np.array([0]), 'input_feature_channel': 2, 'output_feature_channel': 3, 'group': 1, 'layout': 'NHWC', }) return conv_op.create_node([input_reshape_4d_node, conv_filter_const_node], dict(name=input_node.name + "/conv"))
def replace_op(self, graph: Graph, node: Node): if node.has_and_set('inputs_preprocessed'): log.debug('Node "{}" has already been preprocessed'.format( node.soft_get('name'))) return [] # reshape tensor with batch indices to 2d unsqueeze_op = Unsqueeze( graph, {'unsqueeze_dims': np.array([1], dtype=np.int64)}) unsqueeze_node = unsqueeze_op.create_node([node.in_node(2)]) concat_op = Concat( graph, { 'axis': 1, 'name': node.name + '/concat_batch_indices_and_boxes', 'in_ports_count': 2 }) concat_node = concat_op.create_node([unsqueeze_node, node.in_node(1)]) # do not remove edge with crop_size because it is needed in the partial infer graph.remove_edge(node.in_node(1).id, node.id) # input to the CropAndResize contains boxes coordinates in YXYX layout. But IE layer ROIPooling expects # coordinates in the XYXY layout, so convolution is added here to swap coordinates swapped_box_coordinates_node = add_convolution_to_swap_xy_coordinates( graph, concat_node, 5) # reshape locations tensor to 2D so it could be passed to Eltwise which will be converted to ScaleShift reshape_2d_op = Reshape(graph, dict(dim=np.array([-1, 5]))) reshape_2d_node = reshape_2d_op.create_node( [swapped_box_coordinates_node], dict(name=swapped_box_coordinates_node.id + '/reshape_2d_', nchw_layout=True)) graph.create_edge(reshape_2d_node, node, 0, 1) # do not replace any output edge return []
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch): reshape_classes_op = Reshape(graph, {'dim': np.array([0, -1])}) reshape_classes_node = reshape_classes_op.create_node( [match.single_input_node(1)[0]], dict(name='do_reshape_classes')) priors_node = match.single_input_node(2)[0] placeholder = [ Node(graph, node_id) for node_id in graph.nodes() if Node(graph, node_id).op == 'Placeholder' ][0] im_height = placeholder.shape[1] im_width = placeholder.shape[2] # scale prior boxes to the [0, 1] interval priors_scale_const_node = Const( graph, { 'value': np.array( [1 / im_width, 1 / im_height, 1 / im_width, 1 / im_height]) }).create_node([]) priors_scale_node = Eltwise(graph, { 'name': 'scale_priors', 'operation': 'mul' }).create_node([priors_node, priors_scale_const_node]) # calculate prior boxes widths and heights split_node = SplitV(graph, { 'axis': 2, 'size_splits': [1, 1, 1, 1], 'out_ports_count': 4 }).create_node([priors_scale_node]) priors_width_node = __class__._create_sub(graph, split_node, 2, split_node, 0) priors_height_node = __class__._create_sub(graph, split_node, 3, split_node, 1) # concat weights and heights into a single tensor and multiple with the box coordinates regression values concat_width_height_node = Concat(graph, { 'name': 'concat_priors_width_height', 'axis': -1, 'in_ports_count': 4 }).create_node([ priors_width_node, priors_height_node, priors_width_node, priors_height_node ]) applied_width_height_regressions_node = Eltwise(graph, {'name': 'final_regressions', 'operation': 'mul'}). \ create_node([concat_width_height_node, match.single_input_node(0)[0]]) # reshape to 2D tensor as Inference Engine Detection Output layer expects reshape_regression_op = Reshape(graph, {'dim': np.array([0, -1])}) reshape_regression_node = reshape_regression_op.create_node( [applied_width_height_regressions_node], {'name': 'reshape_regression'}) detection_output_op = DetectionOutput( graph, match.custom_replacement_desc.custom_attributes) detection_output_op.attrs['old_infer'] = detection_output_op.attrs[ 'infer'] detection_output_op.attrs['infer'] = __class__.do_infer detection_output_node = detection_output_op.create_node( [reshape_regression_node, reshape_classes_node, priors_scale_node], dict(name=detection_output_op.attrs['type'], clip=1, normalized=1, variance_encoded_in_target=0)) return {'detection_output_node': detection_output_node}