def replace_sub_graph(self, graph: Graph, match: dict): """ Need to find the pattern: SoftmaxActivation -> DetectionOutput DetectionOutput in IE expects flattened input from SoftMax, that is why there is the need to add Flatten layer Parameters ---------- graph : Graph Graph with loaded model. match : dict Patterns which were found in graph structure. """ softmax_activation = match['softmax_activation'] multi_box_detection = match['multi_box_detection'] softmax_activation['axis'] = -1 edge_data = graph.get_edge_data(softmax_activation.id, multi_box_detection.id) out_port = edge_data[0]['out'] in_port = edge_data[0]['in'] graph.remove_edge(softmax_activation.id, multi_box_detection.id) new_reshape_node = create_op_node_with_second_input( graph, Reshape, int64_array([0, -1]), dict(op='Reshape', name=multi_box_detection.name + '/Reshape_'), softmax_activation) graph.create_edge(new_reshape_node, multi_box_detection, in_port=in_port, out_port=out_port)
def add_input_data_to_prior_boxes(graph: Graph, input_names: str = ''): """ PriorBox layer has data input unlike mxnet. Need to add data input to _contrib_MultiBoxPrior for for correct conversion to PriorBox layer. Parameters ---------- graph : Graph Graph with loaded model. """ if not input_names: input_names = ('data', ) else: input_names = input_names.split(',') input_nodes = {} for node in graph.nodes(): node = Node(graph, node) if node.has_valid('op') and node.name in input_names: input_nodes.update({node.id: node}) if len(input_nodes) > 0: for node in graph.nodes(): node = Node(graph, node) if node.has_valid( 'op') and node.op == '_contrib_MultiBoxPrior': node.add_input_port(idx=1) graph.create_edge(list(input_nodes.values())[0], node, out_port=0, in_port=1)
def replace_sub_graph(graph: Graph, match: dict, **kwargs): inputs_dict = {} for u, v, edge_attrs in graph.out_edges(match['queue_deque'].id, data=True): out_port = edge_attrs['out'] shape = match['fifo_queue'].shapes[out_port] if out_port not in inputs_dict: input_op = Parameter(graph, {'shape': shape.copy()}) inputs_dict[out_port] = input_op.create_node([]) graph.create_edge(inputs_dict[out_port], Node(graph, v), edge_attrs['out'], edge_attrs['in'], edge_attrs) graph.remove_node(match['queue_deque'].id) graph.remove_node(match['fifo_queue'].id)
def replace_sub_graph(graph: Graph, match: dict, **kwargs): r""" Usually graph looks like: main_graph ... Result | | image_batch label_batch \ / batch_join / \ placeholder fifo_queue Replacer works for both cases (that's why we have loop - 68 line): label_batch was marked as output there is no label_batch node """ true_placeholder_shape = match['placeholder'].shape placeholder_shape = match['fifo_queue'].shapes[0] placeholder_data_type = match['fifo_queue'].types[0] # in case OOB conversion batch_size placeholder shape is not required # so use a shape specified in FIFOQueueV2 shapes list attribute assert true_placeholder_shape is None or true_placeholder_shape.ndim <= 1 if true_placeholder_shape is not None and true_placeholder_shape.ndim == 1 and len( true_placeholder_shape) > 1: log.warning( 'Placeholder \'{}\' got non 0-dimensional shape {} in FIFOQueue pattern. Placeholder will have the ' 'same shape after folding the pattern instead of {} shape which is original for the network.' ''.format(match['placeholder'].id, true_placeholder_shape, placeholder_shape)) placeholder_shape = true_placeholder_shape placeholder_name = match['fifo_queue'].name graph.erase_node(match['fifo_queue']) graph.erase_node(match['placeholder']) for _, out in match['batch_join'].out_nodes().items(): if out.id != match['image_batch'].id: if out.out_node().op == 'Result': graph.remove_node(out.out_node().id) graph.remove_node(out.id) graph.remove_node(match['batch_join'].id) placeholder = Parameter( graph, { 'name': placeholder_name, 'shape': placeholder_shape, 'data_type': placeholder_data_type }).create_node() graph.create_edge(placeholder, match['image_batch']) log.info( "FIFOQueueV2 pattern was detected. New shape of placeholder {} is {}. Use -b to set batch size if " "needed".format(placeholder.id, placeholder['shape']))
def replace_op(self, graph: Graph, node: Node): if node.has_and_set('inputs_preprocessed'): log.debug('Node "{}" has already been preprocessed'.format( node.soft_get('name'))) return [] # reshape tensor with batch indices to 2d unsqueeze_node = create_op_node_with_second_input( graph, Unsqueeze, int64_array([1]), {'name': node.name + '/Unsqueeze'}, node.in_node(2)) convert_node = Cast( graph, { 'name': unsqueeze_node.name + '/ToFloat', 'dst_type': data_type_str_to_np(graph.graph['cmd_params'].data_type) }).create_node() convert_node.in_port(0).connect(unsqueeze_node.out_port(0)) concat_op = Concat( graph, { 'axis': 1, 'name': node.name + '/concat_batch_indices_and_boxes', 'in_ports_count': 2 }) concat_node = concat_op.create_node([convert_node, node.in_node(1)]) # do not remove edge with crop_size because it is needed in the partial infer graph.remove_edge(node.in_node(1).id, node.id) # input to the CropAndResize contains boxes coordinates in YXYX layout. But IE layer ROIPooling expects # coordinates in the XYXY layout, so convolution is added here to swap coordinates swapped_box_coordinates_node = add_convolution_to_swap_xy_coordinates( graph, concat_node, 5) # reshape locations tensor to 2D so it could be passed to Eltwise which will be converted to ScaleShift reshape_2d_node = create_op_node_with_second_input( graph, Reshape, int64_array([-1, 5]), dict(name=swapped_box_coordinates_node.id + '/reshape_2d_'), swapped_box_coordinates_node) graph.create_edge(reshape_2d_node, node, 0, 1) # do not replace any output edge return []
def replace_sub_graph(self, graph: Graph, match: dict): """ Need to find each occurrence of pattern: transpose -> SoftmaxActivation -> _contrib_MultiBoxDetection remove transpose layer to secure the order of weights in SoftMax to be the same as IE expects IE expects weights to be in following order: class-wise values for each priorbox. priorboxes change the quickest Parameters ---------- graph : Graph Graph with loaded model. match : dict Patterns which were found in graph structure. """ transpose_node = match['transpose'] softmax_activation = match['softmax_activation'] transpose_in_node = transpose_node.in_node(0) graph.remove_edge(transpose_in_node.id, transpose_node.id) graph.remove_edge(transpose_node.id, softmax_activation.id) graph.remove_node(transpose_node.id) graph.create_edge(transpose_in_node, softmax_activation)
def replace_pattern(graph: Graph, match: dict): """ DetectionOutput layer has another order of inputs unlike mxnet. Need to reorder _contrib_MultiBoxDetection inputs for correct conversion to DetectionOutput layer. Parameters ---------- graph : Graph Graph with loaded model. """ multi_box_detection_node = match['multi_box_detection'] conf_node = multi_box_detection_node.in_node(0) loc_node = multi_box_detection_node.in_node(1) conf_edge_data = graph.get_edge_data(conf_node.id, multi_box_detection_node.id) conf_out_port = conf_edge_data[0]['out'] conf_in_port = conf_edge_data[0]['in'] loc_edge_data = graph.get_edge_data(loc_node.id, multi_box_detection_node.id) loc_out_port = loc_edge_data[0]['out'] loc_in_port = loc_edge_data[0]['in'] graph.remove_edge(conf_node.id, multi_box_detection_node.id) graph.remove_edge(loc_node.id, multi_box_detection_node.id) graph.create_edge(loc_node, multi_box_detection_node, in_port=conf_in_port, out_port=conf_out_port) graph.create_edge(conf_node, multi_box_detection_node, in_port=loc_in_port, out_port=loc_out_port)
def load_parallel_component(file_descr, graph: Graph, prev_layer_id): """ Load ParallelComponent of the Kaldi model. ParallelComponent contains parallel nested networks. VariadicSplit is inserted before nested networks. Outputs of nested networks concatenate with layer Concat. :param file_descr: descriptor of the model file :param graph: graph with the topology. :param prev_layer_id: id of the input layers for parallel component layer :return: id of the concat layer - last layer of the parallel component layers """ nnet_count = read_token_value(file_descr, b'<NestedNnetCount>') log.debug( 'Model contains parallel component with {} nested networks'.format( nnet_count)) split_points = [] outputs = [] inputs = [] for i in range(nnet_count): read_token_value(file_descr, b'<NestedNnet>') collect_until_token(file_descr, b'<Nnet>') g = Graph() load_kalid_nnet1_model(g, file_descr, 'Nested_net_{}'.format(i)) # input to nnet1 models is of a rank 1 but we also insert batch_size to 0th axis # 1st axis contains input_size of the nested subnetwork # we split input from the main network to subnetworks input_node = Node(g, 'Parameter') split_points.append(input_node['shape'][1]) g.remove_node(input_node.id) mapping = { node: graph.unique_id(node) for node in g.nodes(data=False) if node in graph } g = nx.relabel_nodes(g, mapping) for val in mapping.values(): g.node[val]['name'] = val graph.add_nodes_from(g.nodes(data=True)) graph.add_edges_from(g.edges(data=True)) sorted_nodes = tuple(nx.topological_sort(g)) outputs.append(Node(graph, sorted_nodes[-1])) inputs.append(Node(graph, sorted_nodes[0])) split_id = graph.unique_id(prefix='NestedNets/VariadicSplit') attrs = { 'out_ports_count': nnet_count, 'size_splits': split_points, 'axis': 1, 'name': split_id } variadic_split_node = AttributedVariadicSplit(graph, attrs).create_node() prev_layer_node = Node(graph, prev_layer_id) prev_layer_node.add_output_port(0) graph.create_edge( prev_layer_node, variadic_split_node, 0, 0, create_edge_attrs(prev_layer_id, variadic_split_node.id, prev_layer_id)) concat_id = graph.unique_id(prefix='Concat') graph.add_node(concat_id, parameters=None, op='concat', kind='op') concat_node = Node(graph, concat_id) # Connect each output of variadic_split_node to each subnetwork's inputs in ParallelComponent # and each subnetwork's output to concat_node for i, (input_node, output_node) in enumerate(zip(inputs, outputs)): output_node.add_output_port(0) concat_node.add_input_port(i) graph.create_edge( output_node, concat_node, 0, i, create_edge_attrs(output_node.id, concat_id, output_node.id, i, 0)) graph.create_edge( variadic_split_node, input_node, i, 0, create_edge_attrs(variadic_split_node.id, input_node.id, variadic_split_node.id, 0, i)) return concat_id