def replace_sub_graph(self, graph: Graph, match: dict): # node that is used to identify this pattern application instance for switching between supported # and not supported LSTMCell sub-graphs; this value will be searched in __class__.instances_supported_by_IE. anchor_node = match[__class__.anchor()] assert anchor_node.has_valid('name'), \ 'LSTMCell anchor node {} does\'t have attribute name; such nodes are not supported.' match['input_op'] = match['concat'].in_node(0) match['input_hidden_state'] = match['concat'].in_node(1) match['input_cell_state'] = match['mul_0'].in_node(0) \ if match['mul_0'].in_node(0).id != match['sigmoid_0'].id else match['mul_0'].in_node(1) pattern_edges = self.pattern()['edges'] pattern_edges.extend([('input_op', 'concat'), ('input_cell_state', 'mul_0'), ('input_hidden_state', 'concat')]) inputs = graph.get_inputs_with_ports( match, pattern_edges, __class__.inputs + __class__.extra_inputs) lstm_op = LSTMCell( graph, dict( name=match['concat'].name + '/LSTMCell', activations=None, )) lstm_node = lstm_op.create_node(inputs) lstm_node['old_infer'] = lstm_node.infer lstm_node.infer = __class__.infer # this node consumes one of the resulting LSTMCell outputs, # it should be removed before reconnecting the nodes, # otherwise it will be reconnected to the new cell output graph.remove_node(match['tanh_1'].id) for i, output in enumerate(__class__.outputs): match[output].replace_node(lstm_node, i) # Because of LSTMCell specification, this layer MUST have 2 outputs. # => we need to create fake consumers for LSTMCell # when this node haven't some outputs. for i in [0, 1]: if i not in lstm_node.out_nodes(): fake_output_node = Result( graph, dict(name=lstm_node.name + "/Output_{}".format(i))) fake_output_node.create_node(inputs=[lstm_node], edge_attrs={ 'out': i, 'in': 0 }) lstm_node['tf'] = True lstm_node['extra_inputs'] = { name: match[name].id for name in __class__.extra_inputs } lstm_node['inputs'] = { name: match[name].id for name in __class__.inputs }
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch): # IE DetectionOutput layer consumes flattened confidences and locations tensors. # That is why we add reshapes before them. locs_node = match.single_input_node(0) conf_node = match.single_input_node(1) prior_boxes_node = match.single_input_node(2) locs_out_nodes = locs_node[0].out_nodes() assert len(locs_out_nodes) == 1 locs_out_node = locs_out_nodes[list(locs_out_nodes.keys())[0]] assert locs_out_node.op == "Result", locs_out_node.op graph.remove_node(locs_out_node.id) conf_out_nodes = conf_node[0].out_nodes() assert len(conf_out_nodes) == 1 conf_out_node = conf_out_nodes[list(conf_out_nodes.keys())[0]] assert conf_out_node.op == "Result", conf_out_node.op graph.remove_node(conf_out_node.id) # reshape operation to flatten confidence tensor const = Const(graph, {'value': int64_array([0, -1])}).create_node() reshape_loc_node = Reshape(graph, {}).create_node( [locs_node, const], dict(name='DetectionOutput_Reshape_loc_')) # reshape operation to flatten confidence tensor reshape_conf_node = Reshape(graph, {}).create_node( [conf_node, const], dict(name='DetectionOutput_Reshape_conf_')) # remove the Result node after the priors node assert prior_boxes_node[0].out_node().op == "Result" graph.remove_node(prior_boxes_node[0].out_node().id) # reshape operation for prior boxes tensor const = Const(graph, {'value': int64_array([1, 2, -1])}).create_node() reshape_priors_node = Reshape(graph, {}).create_node( [prior_boxes_node, const], dict(name='DetectionOutput_Reshape_priors_')) # create Detection Output node with three inputs: locations, confidences and prior boxes detection_output_op = DetectionOutput( graph, match.custom_replacement_desc.custom_attributes) detection_output_node = detection_output_op.create_node( [reshape_loc_node, reshape_conf_node, reshape_priors_node], dict(name=detection_output_op.attrs['type'] + '_')) PermuteAttrs.set_permutation(reshape_priors_node, detection_output_node, None) # create Output node to mark DetectionOutput as a graph output operation output_op = Result(graph) output_op.create_node([detection_output_node], dict(name='sink_')) return {}
def transform_graph(self, graph: Graph, replacement_descriptions: dict): parameter_node = graph.get_op_nodes(op='Parameter')[0] parameter_node['data_type'] = data_type_str_to_np( parameter_node.graph.graph['cmd_params'].data_type) parameter_node.out_port(0).disconnect() # remove existing Result operations to remove unsupported sub-graph graph.remove_nodes_from( [node.id for node in graph.get_op_nodes(op='Result')] + ['detections']) # determine if the op which is a input/final result of mean value and scale applying to the input tensor # then connect it to the input of the first convolution of the model, so we remove the image pre-processing # which includes padding and resizing from the model preprocessing_input_node_id = replacement_descriptions[ 'preprocessing_input_node'] assert preprocessing_input_node_id in graph.nodes, 'The node with name "{}" is not found in the graph. This ' \ 'should be a last node before image normalization and is specified' \ ' in the json file.'.format(preprocessing_input_node_id) preprocessing_input_node = Node(graph, preprocessing_input_node_id) consumer_node = preprocessing_input_node.out_port( 0).get_connection().get_destination().node consumer_node.in_port(0).get_connection().set_source( parameter_node.out_port(0)) preprocessing_output_node_id = replacement_descriptions[ 'preprocessing_output_node'] assert preprocessing_output_node_id in graph.nodes, 'The node with name "{}" is not found in the graph. This ' \ 'node should provide scaled image output and is specified' \ ' in the json file.'.format(preprocessing_output_node_id) preprocessing_output_node = Node(graph, preprocessing_output_node_id) preprocessing_output_node.out_port(0).disconnect() convolution_nodes = [ n for n in graph.pseudo_topological_sort() if n.soft_get('type') == 'Convolution' ] convolution_nodes[0].in_port(0).get_connection().set_source( preprocessing_output_node.out_port(0)) # create prior boxes (anchors) generator aspect_ratios = replacement_descriptions['aspect_ratios'] assert len(aspect_ratios) % 2 == 0 aspect_ratios = list(zip(aspect_ratios[::2], aspect_ratios[1::2])) priors_generator = self.AnchorGenerator( min_level=int(replacement_descriptions['min_level']), aspect_ratios=aspect_ratios, num_scales=int(replacement_descriptions['num_scales']), anchor_scale=replacement_descriptions['anchor_scale']) prior_boxes = [] for i in range(100): inp_name = 'box_net/box-predict{}/BiasAdd'.format('_%d' % i if i else '') if inp_name not in graph: break widths, heights = priors_generator.get(i) prior_box_op = PriorBoxClusteredOp( graph, { 'width': np.array(widths), 'height': np.array(heights), 'clip': 0, 'flip': 0, 'variance': replacement_descriptions['variance'], 'offset': 0.5 }) prior_boxes.append( prior_box_op.create_node( [Node(graph, inp_name), parameter_node])) # concatenate prior box operations concat_prior_boxes = Concat(graph, {'axis': -1}).create_node() for idx, node in enumerate(prior_boxes): concat_prior_boxes.add_input_port(idx) concat_prior_boxes.in_port(idx).connect(node.out_port(0)) conf = Sigmoid(graph, dict(name='concat/sigmoid')).create_node( [Node(graph, 'concat')]) reshape_size_node = Const(graph, { 'value': int64_array([0, -1]) }).create_node([]) logits = Reshape(graph, dict(name=conf.name + '/Flatten')).create_node( [conf, reshape_size_node]) deltas = Reshape(graph, dict(name='concat_1/Flatten')).create_node( [Node(graph, 'concat_1'), reshape_size_node]) # revert convolution boxes prediction weights from yxYX to xyXY (convolutions share weights and bias) weights = Node(graph, 'box_net/box-predict/pointwise_kernel') weights.value = weights.value.reshape(-1, 4)[:, [1, 0, 3, 2]].reshape( weights.shape) bias = Node(graph, 'box_net/box-predict/bias') bias.value = bias.value.reshape(-1, 4)[:, [1, 0, 3, 2]].reshape(bias.shape) detection_output_node = DetectionOutput( graph, dict( name='detections', num_classes=int(replacement_descriptions['num_classes']), share_location=1, background_label_id=int( replacement_descriptions['num_classes']) + 1, nms_threshold=replacement_descriptions['nms_threshold'], confidence_threshold=replacement_descriptions[ 'confidence_threshold'], top_k=100, keep_top_k=100, code_type='caffe.PriorBoxParameter.CENTER_SIZE', )).create_node([deltas, logits, concat_prior_boxes]) output_op = Result(graph, dict(name='output')) output_op.create_node([detection_output_node])