def replace_pattern(self, graph: Graph, match: dict): node = match['node'] node_name = node.soft_get('name', node.id) connected_ports = [port for port in node.in_ports().values() if not port.disconnected()] if len(connected_ports) == 2: axis = node.in_port(1).data.get_value() else: axis = node.axis assert axis is not None, 'The "axis" should be defined for node "{}"'.format(node_name) assert node.has_and_set('output_type'), 'The data type is not set for node "{}"'.format(node_name) topk_mode = 'max' if node.op == 'ArgMax' else 'min' topk_node = TopK(graph, {'axis': axis, 'mode': topk_mode, 'sort': 'index', 'remove_values_output': node.has_and_set('remove_values_output'), 'index_element_type': node.output_type}).create_node() node.in_port(0).get_connection().set_destination(topk_node.in_port(0)) if node.has_and_set('out_max_val'): # in this mode the ArgMax produces tuples (max_ind, max_value) concat_node = Concat(graph, {'axis': 1, 'name': node.name + '/Concat'}).create_node() concat_node.add_input_port(0, skip_if_exist=True) concat_node.add_input_port(1, skip_if_exist=True) topk_node.out_port(0).connect(concat_node.in_port(1)) # indices topk_node.out_port(1).connect(concat_node.in_port(0)) # values if not node.out_port(0).disconnected(): node.out_port(0).get_connection().set_source(concat_node.out_port(0)) else: if not node.out_port(0).disconnected(): node.out_port(0).get_connection().set_source(topk_node.out_port(1)) topk_node.in_port(1).connect(Const(graph, {'name': node.soft_get('name') + '/TopK', 'value': node.top_k}).create_node().out_port(0)) graph.remove_nodes_from([node.id, node.out_node(0).id])
def create_ss_interval_border(graph: Graph, slice_border_port: Port, shape: np.ndarray, axes: np.ndarray, node_name: str): """ This function creates "begin"/"end" parameters for the StridedSlice based on Slice's "starts"/"ends" :param graph: graph to operate on. :param slice_border_port: node output port that provides "starts"/"ends" values for the Slice. :param shape: input shape of the Slice :param axes: axes that "starts" and "ends" apply to :param node_name: Slice node name :return: Concat node that forms "begin"/"end" values for the StridedSlice """ # the value for 'starts' or 'ends' might be maximum/minimum possible value of int64. This # value must be converted to maximum/minimum of int32 because such big values do not fit into the int32 which is # supported by the StridedSlice layer clamp = create_op_with_const_inputs(graph, Clamp, port_value_dict={ 1: np.iinfo(np.int32).min, 2: np.iinfo(np.int32).max }, op_attrs=dict(name=node_name + '/Clamp')) clamp.in_port(0).connect(slice_border_port) # we have to convert "starts"/"ends" values from the network to one data type with constant values that are created # here to prevent type errors in Concat node cast = Cast(graph, dict(name=node_name + '/CastToI64', dst_type=np.int64)).create_node() cast.in_port(0).connect(clamp.out_port(0)) concat = Concat(graph, dict(name=node_name + '/Concat', axis=0)).create_node() for value_idx, port_idx in enumerate(axes): concat.add_input_port(port_idx) # "axes" may not be sorted, so we need to split "starts"/"ends" values and connect each value to the correct # Concat input port value = create_op_with_const_inputs( graph, Gather, port_value_dict={ 1: int64_array([value_idx]), 2: int64_array(0) }, op_attrs={'name': node_name + '/Gather'}) cast.out_port(0).connect(value.in_port(0)) value.out_port(0).connect(concat.in_port(port_idx)) for port_idx in range(len(shape)): if not concat.is_in_port_connected(port_idx): concat.add_input_port(port_idx) # This border value would be ignored in StridedSlice because of the begin_mask\end_mask const = Const( graph, dict(name=node_name + '/Const', value=int64_array([0]))).create_node() const.out_port(0).connect(concat.in_port(port_idx)) return concat
def placeholder_scales(self, placeholder: Node): """ Helper function to get scales for prior boxes out of input image size: [1 / im_width, 1 / im_height, 1 / im_width, 1 / im_height] """ graph = placeholder.graph name = placeholder.soft_get('name', placeholder.id) shape_value = placeholder.soft_get('shape', None) assert shape_value is not None, \ "[ {} replacer ] Placeholder `{}` should have shape attribute".format(self.replacement_id, name) assert isinstance(shape_value, np.ndarray), \ "[ {} replacer ] Placeholder `{}` shape attribute should be np.ndarray".format(self.replacement_id, name) assert shape_value.size == 4, \ "[ {} replacer ] Placeholder `{}` should be 4D. Shape: {}".format(self.replacement_id, name, shape_value) shape = Shape(graph, {'name': 'input_image_shape'}).create_node() shape.in_port(0).connect(placeholder.out_port(0)) begin = Const(graph, {'value': int64_array([1])}).create_node() end = Const(graph, {'value': int64_array([3])}).create_node() stride = Const(graph, {'value': int64_array([1])}).create_node() spatial = StridedSlice(graph, {'name': name + '/get_h_w', 'begin_mask': int64_array([1]), 'end_mask': int64_array([1]), 'new_axis_mask': int64_array([0]), 'shrink_axis_mask': int64_array([0]), 'ellipsis_mask': int64_array([0])}).create_node() spatial.in_port(0).connect(shape.out_port(0)) spatial.in_port(1).connect(begin.out_port(0)) spatial.in_port(2).connect(end.out_port(0)) spatial.in_port(3).connect(stride.out_port(0)) power = Const(graph, {'value': float32_array([-1.])}).create_node() spatial_scale = Pow(graph, {}).create_node() spatial_scale.in_port(0).connect(spatial.out_port(0)) spatial_scale.in_port(1).connect(power.out_port(0)) # Power `type_infer` requires inputs to have equal data type convert_to_fp32 = Cast(graph, {'dst_type': np.float32}).create_node() spatial_scale.in_port(0).get_connection().insert_node(convert_to_fp32) order = Const(graph, {'value': int64_array([1, 0])}).create_node() axis_const = Const(graph, {'value': int64_array(0)}).create_node() reverse = Gather(graph, {}).create_node() reverse.in_port(0).connect(spatial_scale.out_port(0)) reverse.in_port(1).connect(order.out_port(0)) axis_const.out_port(0).connect(reverse.in_port(2)) priors_scale_node = Concat(graph, {'axis': 0, 'in_ports_count': 2}).create_node() priors_scale_node.add_input_port(0, skip_if_exist=True) priors_scale_node.add_input_port(1, skip_if_exist=True) priors_scale_node.in_port(0).connect(reverse.out_port(0)) priors_scale_node.in_port(1).connect(reverse.out_port(0)) return priors_scale_node
def replace_tdnn(self, graph: Graph, tdnn_node: Node): tdnn_name = tdnn_node.soft_get('name', tdnn_node.id) concat_node = Concat(graph, {'axis': 1}).create_node() rename_nodes([(tdnn_node, tdnn_name + '/to_be_removed'), (concat_node, tdnn_name)]) for offset_ind, t in enumerate(tdnn_node['time_offsets']): concat_node.add_input_port(offset_ind) if t != 0: memory_name = tdnn_name + '/MemoryOffset/' + str(abs(t)) memoryoffset_node = MemoryOffset( graph, { 'name': memory_name, 't': t, 'pair_name': memory_name + '_out', 'has_default': False, 'splitted': False }).create_node() tdnn_node.in_port(0).get_source().connect( memoryoffset_node.in_port(0)) memoryoffset_node.out_port(0).connect( concat_node.in_port(offset_ind)) else: # 0 time delay is not allowed in IE, it's meaningless # if time offset is 0 then connect input of tdnncomponent directly to Concat without memoryoffset tdnn_node.in_port(0).get_source().connect( concat_node.in_port(offset_ind)) weights = tdnn_node['weights'] fc_inputs = {1: weights} bias_term = False if tdnn_node.has_valid('biases'): assert len(tdnn_node['biases']) == weights.shape[0] fc_inputs.update({2: tdnn_node['biases']}) bias_term = True fc_node = create_op_with_const_inputs( graph, FullyConnected, fc_inputs, { 'name': tdnn_name + '/FC', 'out-size': weights.shape[0], 'transpose_weights': True, 'bias_term': bias_term }) concat_node.out_port(0).connect(fc_node.in_port(0)) tdnn_node.in_port(0).disconnect() tdnn_node.out_port(0).get_connection().set_source(fc_node.out_port(0))
def new_shape_node_from_shape_nodes(input_shape_nodes: list): """ The function returns a node producing 1D tensor with concatenated shapes produced by nodes from "input_shape_nodes" :param input_shape_nodes: list of nodes producing 1D tensors :return: the node producing concatenated values of nodes from the "input_shape_nodes" """ assert len(input_shape_nodes ) > 0, 'The list of input shape nodes should be non-empty' new_shape_node = Concat( input_shape_nodes[0].graph, { 'axis': 0, 'name': input_shape_nodes[0].soft_get('name', input_shape_nodes[0].id) + '/shapes_concat' }).create_node() for ind, input_node in enumerate(input_shape_nodes): new_shape_node.add_input_port(ind) new_shape_node.in_port(ind).connect(input_node.out_port(0)) return new_shape_node
def transform_graph(self, graph: Graph, replacement_descriptions: dict): parameter_node = graph.get_op_nodes(op='Parameter')[0] parameter_node['data_type'] = data_type_str_to_np( parameter_node.graph.graph['cmd_params'].data_type) # remove existing Result operations to remove unsupported sub-graph graph.remove_nodes_from( [node.id for node in graph.get_op_nodes(op='Result')] + ['detections']) # determine if the op which is a input/final result of mean value and scale applying to the input tensor # then connect it to the input of the first convolution of the model, so we remove the image pre-processing # which includes padding and resizing from the model preprocessing_input_node_id = replacement_descriptions[ 'preprocessing_input_node'] assert preprocessing_input_node_id in graph.nodes, 'The node with name "{}" is not found in the graph. This ' \ 'should be a last node before image normalization and is specified' \ ' in the json file.'.format(preprocessing_input_node_id) preprocessing_input_node = Node(graph, preprocessing_input_node_id) consumer_node = preprocessing_input_node.out_port( 0).get_connection().get_destination().node consumer_node.in_port(0).get_connection().set_source( parameter_node.out_port(0)) preprocessing_output_node_id = replacement_descriptions[ 'preprocessing_output_node'] assert preprocessing_output_node_id in graph.nodes, 'The node with name "{}" is not found in the graph. This ' \ 'node should provide scaled image output and is specified' \ ' in the json file.'.format(preprocessing_output_node_id) preprocessing_output_node = Node(graph, preprocessing_output_node_id) preprocessing_output_node.out_port(0).disconnect() convolution_nodes = [ n for n in graph.pseudo_topological_sort() if n.soft_get('type') == 'Convolution' ] convolution_nodes[0].in_port(0).get_connection().set_source( preprocessing_output_node.out_port(0)) # create prior boxes (anchors) generator aspect_ratios = replacement_descriptions['aspect_ratios'] assert len(aspect_ratios) % 2 == 0 aspect_ratios = list(zip(aspect_ratios[::2], aspect_ratios[1::2])) priors_generator = self.AnchorGenerator( min_level=int(replacement_descriptions['min_level']), aspect_ratios=aspect_ratios, num_scales=int(replacement_descriptions['num_scales']), anchor_scale=replacement_descriptions['anchor_scale']) prior_boxes = [] for i in range(100): inp_name = 'box_net/box-predict{}/BiasAdd'.format('_%d' % i if i else '') if inp_name not in graph: break widths, heights = priors_generator.get(i) prior_box_op = PriorBoxClusteredOp( graph, { 'width': mo_array(widths), 'height': mo_array(heights), 'clip': 0, 'flip': 0, 'variance': replacement_descriptions['variance'], 'offset': 0.5 }) prior_boxes.append( prior_box_op.create_node( [Node(graph, inp_name), parameter_node])) # concatenate prior box operations concat_prior_boxes = Concat(graph, {'axis': -1}).create_node() for idx, node in enumerate(prior_boxes): concat_prior_boxes.add_input_port(idx) concat_prior_boxes.in_port(idx).connect(node.out_port(0)) conf = Sigmoid(graph, dict(name='concat/sigmoid')).create_node( [Node(graph, 'concat')]) reshape_size_node = Const(graph, { 'value': int64_array([0, -1]) }).create_node([]) logits = Reshape(graph, dict(name=conf.name + '/Flatten')).create_node( [conf, reshape_size_node]) deltas = Reshape(graph, dict(name='concat_1/Flatten')).create_node( [Node(graph, 'concat_1'), reshape_size_node]) # revert convolution boxes prediction weights from yxYX to xyXY (convolutions share weights and bias) weights = Node(graph, 'box_net/box-predict/pointwise_kernel') weights.value = weights.value.reshape(-1, 4)[:, [1, 0, 3, 2]].reshape( weights.shape) bias = Node(graph, 'box_net/box-predict/bias') bias.value = bias.value.reshape(-1, 4)[:, [1, 0, 3, 2]].reshape(bias.shape) detection_output_node = DetectionOutput( graph, dict( name='detections', share_location=1, background_label_id=int( replacement_descriptions['num_classes']) + 1, nms_threshold=replacement_descriptions['nms_threshold'], confidence_threshold=replacement_descriptions[ 'confidence_threshold'], top_k=100, keep_top_k=100, code_type='caffe.PriorBoxParameter.CENTER_SIZE', )).create_node([deltas, logits, concat_prior_boxes]) output_op = Result(graph, dict(name='output')) output_op.create_node([detection_output_node])