def replace_pattern(graph: Graph, match: dict): split = match['split'] # Check that we need to add fake output (case when input.shape[axis] != sum(outputs.shape[axis]) axis = split.axis input_shape = split.in_port(0).data.get_shape()[axis] output_shape = sum([split.out_node(port).shape[axis] for port in split.out_nodes()]) # In such case we don't need to do anything if input_shape == output_shape: return # Adding fake outputs n_parts = int(input_shape/split.size_splits[0]) part_shape = split.in_port(0).data.get_shape().copy() part_shape[axis] = split.size_splits[0] out_ports = split.out_ports() for i in range(n_parts): if i in out_ports and not split.out_port(i).disconnected(): continue if i not in out_ports: split.add_output_port(i) output = Result(graph).create_node(attrs={'name': split.name + '/Fake_output_{}/'.format(i)}) split.out_port(i).connect(output.in_port(0)) output.in_port(0).data.set_shape(part_shape)
def replace_pattern(self, graph: Graph, match: dict): node = match['pooling'] node.type = 'MaxPool' del node['pool_method'] if 'exclude_pad' in node: del node['exclude_pad'] # adding missed outputs for MaxPool node if node.out_port(0).disconnected(): output = Result( node.graph, { 'name': node.name + '/Result_port_0/', 'keep_output_port': node.has_and_set('remove_values_output') }).create_node() node.out_port(0).get_connection().set_destination( output.in_port(0)) if node.out_port(1).disconnected(): output = Result( node.graph, { 'name': node.name + '/Result_port_1/', 'keep_output_port': node.has_and_set('remove_values_output') }).create_node() node.out_port(1).get_connection().set_destination( output.in_port(0))
def replace_pattern(graph: Graph, match: dict): node = match['op'] pair_node = Node(graph, node.pair_name) if node.t >= 0: raise Error('Does not support IfDefined with t > 0') if node.in_port(0).get_source() is not None: input_port = node.in_port(0).get_source() op_output_id = node.out_port(0).get_destination().node.id out_port = pair_node.out_port(0) node_name = node.name pair_name = pair_node.name else: input_port = pair_node.in_port(0).get_source() op_output_id = pair_node.out_port(0).get_destination().node.id out_port = node.out_port(0) node_name = pair_node.name pair_name = node.name in_shape = input_port.data.get_shape() node_t = abs(node.t) init_value_memory_out = create_zero_value_with_batch_from_input(input_port, in_shape[1]*node_t) memory_out = ReadValue(graph, {'name': pair_name, 'variable_id': node_name+pair_name}).create_node() init_value_memory_out.out_port(0).connect(memory_out.in_port(0)) if node_t > 1: crop_concat = Crop(graph, {'name': 'Memory_crop', 'dim': np.array([in_shape[1]*(node_t-1)]), 'offset': np.array([in_shape[1]]), 'axis': np.array([1])}).create_node() memory_out.out_port(0).connect(crop_concat.in_port(0)) concat = Concat(graph, {'name': 'Memory_concat'}).create_node() concat.add_sequence_of_ports('in', range(2)) crop_concat.out_port(0).connect(concat.in_port(0)) concat.in_port(1).connect(input_port) memory_in = Assign(graph, {'name': node_name, 'variable_id': node_name + pair_name}).create_node() concat.out_port(0).connect(memory_in.in_port(0)) out = Result(graph, {'name': 'Memory_output'}).create_node() memory_in.out_port(0).connect(out.in_port(0)) crop_out = Crop(graph, {'name': 'Memory_crop_out', 'dim': np.array([in_shape[1]]), 'offset': np.array([0]), 'axis': np.array([1])}).create_node() memory_out.out_port(0).connect(crop_out.in_port(0)) out_port.get_connection().set_source(crop_out.out_port(0)) else: memory_in = Assign(graph, {'name': node_name, 'variable_id': node_name + pair_name}).create_node() memory_in.in_port(0).connect(input_port) out = Result(graph, {'name': 'Memory_output'}).create_node() memory_in.out_port(0).connect(out.in_port(0)) out_port.get_connection().set_source(memory_out.out_port(0)) graph.remove_node(op_output_id) graph.remove_node(node.id) graph.remove_node(pair_node.id)
def find_and_replace_pattern(self, graph: Graph): for ctc_greedy_decoder_tf in graph.get_op_nodes(op='CTCGreedyDecoderSeqLen', output_sparse_format=True): ctc_greedy_decoder_tf_name = ctc_greedy_decoder_tf.soft_get('name', ctc_greedy_decoder_tf.id) # TF CTCGreedyDecoder have 4 output tensors. If any of them connected to not Result operation then # transformation in not applicable for port_num in ctc_greedy_decoder_tf.out_ports(): if not ctc_greedy_decoder_tf.out_port(port_num).disconnected()\ and ctc_greedy_decoder_tf.out_port(port_num).get_destination().node.soft_get('op') != 'Result': return # If the first and second output are not connected to Result operations - # create Result operation and connect it to appropriate output if ctc_greedy_decoder_tf.out_port(0).disconnected(): first_result = Result(graph, {'name': ctc_greedy_decoder_tf_name + '/decoded_classes'} ).create_node() ctc_greedy_decoder_tf.out_port(0).connect(first_result.in_port(0)) if ctc_greedy_decoder_tf.out_port(1).disconnected(): second_result = Result(graph, {'name': ctc_greedy_decoder_tf_name + '/seq_lengths_output'} ).create_node() ctc_greedy_decoder_tf.out_port(1).connect(second_result.in_port(0)) # For normalizing input channel needs to transpose input data from [T, N, C] to [N, T, C] # which supported CTCGreedyDecoderSeqLen op. log.warning('Found TF CTCGreedyDecoder operation at the end of network. ' 'PLEASE NOTE, appropriate network output operation CTCGreedyDecoderSeqLen {} ' 'will have dense format, not sparse format!'.format(ctc_greedy_decoder_tf_name)) ctc_data_permute = create_op_with_const_inputs(graph, Transpose, {1: int64_array([1, 0, 2])}, {'name': ctc_greedy_decoder_tf_name + '/ctc_data_permute'}) assert ctc_greedy_decoder_tf.has_valid('merge_repeated'), \ 'The CTCGreedyDecoderSeqLen node "{}" misses "merge_repeated" attribute'.format( ctc_greedy_decoder_tf_name) ctc_greedy_decoder_tf.in_port(0).get_source().connect(ctc_data_permute.in_port(0)) ctc_greedy_decoder_tf.in_port(0).disconnect() ctc_data_permute.out_port(0).connect(ctc_greedy_decoder_tf.in_port(0)) del ctc_greedy_decoder_tf['output_sparse_format'] for port_num in [2, 3]: # MO CTCGreedyDecoderSeqLen may have 2 outputs if port_num in ctc_greedy_decoder_tf.out_ports(): if not ctc_greedy_decoder_tf.out_port(port_num).disconnected(): ctc_greedy_decoder_tf.out_port(port_num).disconnect()
def find_and_replace_pattern(self, graph: Graph): for offset_node in graph.get_op_nodes(op='MemoryOffset', splitted=False): paired_node = MemoryOffset( graph, { 'name': offset_node.pair_name, 'splitted': True, 'pair_name': offset_node.id, 't': offset_node.t, 'has_default': offset_node.has_default }).create_node() offset_node['splitted'] = True offset_node.out_port(0).get_connection().set_source( paired_node.out_port(0)) res_node = Result(graph, { 'name': offset_node.id + "_output" }).create_node() offset_node.out_port(0).connect(res_node.in_port(0)) # If 'element_size' is previously copied from Parameter of from node with defined dim if offset_node.has_valid('element_size'): paired_node['element_size'] = offset_node['element_size'] # Copy shape from previous node. Typically (but not always) for TDNN blocks this is the case else: paired_node['element_size'] = offset_node.in_port( 0).data.get_shape()[1]
def add_output_in_body(node, port_num, cur_graph, cur_max_layer_id, tracks, track_index, add_unsqueeze=True): port = node.out_port(port_num) if add_unsqueeze: unsq_name = port.node.soft_get('name', port.node.id) + "/Unsqueeze" unsq_node = create_op_node_with_second_input(cur_graph, Unsqueeze, int64_array([0]), {'name': unsq_name}) port.connect(unsq_node.in_port(0)) unsq_node['internal_layer_id'] = cur_max_layer_id + 1 cur_max_layer_id += 1 tracks.insert(track_index, {'node': unsq_node, 'graph': cur_graph}) port = unsq_node.out_port(0) out_name = port.node.soft_get('name', port.node.id) + ":" + str(port_num) res_node = Result(cur_graph, {'name': out_name}).create_node() port.connect(res_node.in_port(0)) res_node['internal_layer_id'] = cur_max_layer_id + 1 cur_max_layer_id += 1 tracks.insert(track_index, {'node': res_node, 'graph': cur_graph}) return res_node
def replace_pattern(graph: Graph, match: dict): offset_node = match['mem_offset'] paired_node = MemoryOffset(graph, {'name': offset_node.pair_name, 'splitted': True, 'pair_name': offset_node.id, 't': offset_node.t, 'has_default': offset_node.has_default}).create_node() offset_node['splitted'] = True offset_node.out_port(0).get_connection().set_source(paired_node.out_port(0)) res_node = Result(graph, {'name': offset_node.id+"_output"}).create_node() offset_node.out_port(0).connect(res_node.in_port(0))
def normalize_outputs(node: Node): """ This function adds missed outputs for TopK node. """ if node.out_port(0).disconnected(): output = Result( node.graph, { 'name': node.name + '/Result_port_0/', 'remove_from_xml': node.has_and_set('remove_values_output') }).create_node() node.out_port(0).get_connection().set_destination( output.in_port(0)) if node.out_port(1).disconnected(): output = Result(node.graph, { 'name': node.name + '/Result_port_1/' }).create_node() node.out_port(1).get_connection().set_destination( output.in_port(0))
def assign_add_output_result(op: Node): """ Function adds necessary output result node for Assign node :param op: :return: """ assert op.soft_get('type') == 'Assign', 'Wrong operation type, {} instead of Assign!' \ ''.format(op.soft_get('type')) tmp_result = Result(op.graph, {'name': op.soft_get('name', op.id) + '/Result'}).create_node() op.out_port(0).connect(tmp_result.in_port(0))
def split_normalize_outputs(node: Node): if node.has_valid('out_ports_count') and len( node.out_edges()) < node.out_ports_count: for p in range(node.out_ports_count): if p not in node.out_ports(): node.add_output_port(p) if node.out_port(p).disconnected(): res_node = Result( node.graph, { 'name': node.name + '/Fake_output_{}/'.format(p), 'keep_output_port': True }).create_node() node.out_port(p).connect(res_node.in_port(0))
def replace_pattern(graph: Graph, match: dict): node = match['op'] if node.has_valid('out_ports_count') and len( node.out_edges()) < node.out_ports_count: for p in range(node.out_ports_count): if p not in node.out_ports(): node.add_output_port(p) if node.out_port(p).disconnected(): res_node = Result( graph, { 'name': node.name + '/Fake_output_{}/'.format(p) }).create_node() node.out_port(p).connect(res_node.in_port(0))
def normalize_outputs(node: Node): if node.has_valid('out_ports_count') and len( node.out_edges()) < node.out_ports_count: from mo.ops.result import Result # Import is here to avoid circular import error for p in range(node.out_ports_count): if p not in node.out_ports(): node.add_output_port(p) if node.out_port(p).disconnected(): res_node = Result( node.graph, { 'name': node.name + '/Fake_output_{}/'.format(p), 'keep_output_port': True }).create_node() node.out_port(p).connect(res_node.in_port(0))
def replace_pattern(graph: Graph, match: dict): node = match['result'] is_scalar = graph.graph['cmd_params'].generate_experimental_IR_V10 reshape = create_op_node_with_second_input( graph, Reshape, int64_array([]) if is_scalar else int64_array([1]), {'override_output_shape': True}) node.in_port(1).get_connection().insert_node(reshape) if node.out_port(0).disconnected(): output = Result( graph, { 'name': node.name + '/Result_port_0/', 'remove_from_xml': node.has_and_set('remove_values_output') }).create_node() node.out_port(0).get_connection().set_destination( output.in_port(0)) if node.out_port(1).disconnected(): output = Result(graph, { 'name': node.name + '/Result_port_1/' }).create_node() node.out_port(1).get_connection().set_destination( output.in_port(0))
def replace_pattern(graph: Graph, match: dict): node = match['result'] k = node.in_port(1).data.get_value() if not isinstance(k, np.ndarray): node.in_port(1).data.set_value(int64_array([k])) elif k.ndim == 0: node.in_port(1).data.set_value(int64_array([k.item()])) else: log.debug( 'The "k" input to the TopK layer "{}" is already 1D'.format( node.soft_get('name'))) if node.out_port(0).disconnected(): output = Result(graph, { 'name': node.name + '/Result_port_0/' }).create_node() node.out_port(0).get_connection().set_destination( output.in_port(0)) if node.out_port(1).disconnected(): output = Result(graph, { 'name': node.name + '/Result_port_1/' }).create_node() node.out_port(1).get_connection().set_destination( output.in_port(0))
def split_offset(offset_node: Node): paired_node = MemoryOffset( offset_node.graph, { 'name': offset_node.pair_name, 'splitted': True, 'pair_name': offset_node.id, 'element_size': offset_node['element_size'], 't': offset_node.t, 'has_default': offset_node.has_default }).create_node() offset_node['splitted'] = True offset_node.out_port(0).get_connection().set_source( paired_node.out_port(0)) res_node = Result(offset_node.graph, { 'name': offset_node.id + '_output' }).create_node() offset_node.out_port(0).connect(res_node.in_port(0))
def test_leaky_relu_mul_multiple_consumers(self): # multiple consumers of Mul operation graph = build_graph_with_edge_attrs(nodes, edges, {}) additional_result = Result(graph, {'name': 'result_2'}).create_node() Node(graph, 'mul').out_port(0).connect(additional_result.in_port(0)) ref_nodes = { **regular_op_with_shaped_data('input', shape, { 'type': 'Parameter', 'op': 'Parameter' }), **regular_op_with_shaped_data('mul', shape, { 'type': 'Multiply', 'name': 'mul' }), **regular_op_with_shaped_data('max', shape, { 'type': 'Maximum', 'name': 'final_max' }), **valued_const_with_data('const', float_array([0.5])), **regular_op_with_shaped_data('leaky_relu', shape, { 'type': 'LeakyReLU', 'name': 'max_final', 'negative_slope': None }), **result('result'), **result('result_2') } ref_edges = [ *connect('input:0', '0:mul'), *connect('const', '1:mul'), *connect('max:0', 'result'), *connect('mul:0', 'result_2'), *connect_data('input', 'leaky_relu'), *connect('leaky_relu', 'result') ] graph_ref = build_graph_with_edge_attrs(ref_nodes, ref_edges) LeakyReLUFusion().find_and_replace_pattern(graph) graph.clean_up() (flag, resp) = compare_graphs(graph, graph_ref, 'result') self.assertTrue(flag, resp) (flag, resp) = compare_graphs(graph, graph_ref, 'result_2') self.assertTrue(flag, resp)
def replace_pattern(graph: Graph, match: dict): node = match['op'] if node.name == 'iteration_number_out': return # calculate length of context when state of inference becomes meaningful inputs = [] for n in graph.get_op_nodes(**{'op': 'Parameter'}): inputs.append(n) in_nodes = [] for inp in inputs: for ins in inp.out_port(0).get_destinations(): in_nodes.append(ins.node.name) context_len = 1 try: subgraph = invert_sub_graph_between_nodes( graph, [node.in_port(0).get_source().node.name], in_nodes) except Error: return for n in subgraph: n_node = Node(graph, n) if n_node.kind == 'op' and n_node.op == 'Splice': context_len += len(n_node.context) - 1 if context_len == 1: return in_node_port = node.in_port(0).get_source() in_node_shape = node.in_port(0).data.get_shape() node.in_port(0).disconnect() # add Select before saving state to avoid saving garbage select_node = Select(graph, { 'name': 'select_' + node.name }).create_node() zero_else = Const(graph, { 'name': 'zero_else', 'value': np.zeros(in_node_shape) }).create_node() select_node.in_port(1).connect(in_node_port) select_node.in_port(2).connect(zero_else.out_port(0)) # check if we have already appropriate iteration counter existing_counters = find_pattern_matches( graph, nodes=[('mem_in', dict(op='ReadValue')), ('mem_in_data', dict(shape=int64_array([context_len]))), ('crop_mem_in', dict(op='Crop', axis=int64_array([1]), offset=int64_array([1]), dim=int64_array([context_len - 1]))), ('crop_mem_in_data', dict()), ('concat', dict(op='Concat', axis=1)), ('concat_data', dict()), ('const_1', dict(op='Const')), ('const_1_data', dict()), ('mem_out', dict(op='Assign')), ('crop_out', dict(op='Crop', axis=int64_array([1]), offset=int64_array([0]), dim=int64_array([1]))), ('crop_out_data', dict()), ('select', dict(op='Select'))], edges=[('mem_in', 'mem_in_data'), ('mem_in_data', 'crop_mem_in'), ('crop_mem_in', 'crop_mem_in_data'), ('crop_mem_in_data', 'concat', { 'in': 0 }), ('const_1', 'const_1_data'), ('const_1_data', 'concat', { 'in': 1 }), ('concat', 'concat_data'), ('concat_data', 'mem_out'), ('concat_data', 'crop_out'), ('crop_out', 'crop_out_data'), ('crop_out_data', 'select')]) counter_match = next(existing_counters, None) if counter_match is not None: ones = Node(graph, inverse_dict(counter_match)['const_1']) input_port = Node( graph, inverse_dict(counter_match)['crop_out']).out_port(0) else: init_value_mem_out = create_zero_value_with_batch_from_input( in_node_port, context_len, np.int32) mem_out = ReadValue( graph, { 'name': 'iteration_number', 'variable_id': 'iteration_' + node.name }).create_node() mem_out.in_port(0).connect(init_value_mem_out.out_port(0)) cut_first = Crop( graph, { 'name': 'cut_first', 'axis': int64_array([1]), 'offset': int64_array([1]), 'dim': int64_array([context_len - 1]) }).create_node() cut_first.in_port(0).connect(mem_out.out_port(0)) ones = Const(graph, { 'name': 'ones', 'value': np.ones([1, 1], dtype=np.int32) }).create_node() concat = Concat(graph, { 'name': 'concat_ones', 'in_ports_count': 2, 'axis': 1 }).create_node() concat.in_port(0).connect(cut_first.out_port(0)) concat.in_port(1).connect(ones.out_port(0)) mem_in = Assign( graph, { 'name': 'iteration_number_out', 'variable_id': 'iteration_' + node.name }).create_node() mem_in.in_port(0).connect(concat.out_port(0)) res = Result(graph, {}).create_node() mem_in.out_port(0).connect(res.in_port(0)) cut_last = Crop( graph, { 'name': 'cut_last', 'axis': int64_array([1]), 'offset': int64_array([0]), 'dim': int64_array([1]) }).create_node() cut_last.in_port(0).connect(concat.out_port(0)) input_port = cut_last.out_port(0) # Check if data from memory is 1 # if it is True, we have correct data and should proceed with saving it to memory # else we have not gathered context and have garbage here, shouldn't change initial state of memory cast_in = Equal(graph, { 'name': input_port.node.name + '/cast_to_bool' }).create_node() cast_in.in_port(0).connect(ones.out_port(0)) cast_in.in_port(1).connect(input_port) select_node.in_port(0).connect(cast_in.out_port(0)) select_node.out_port(0).connect(node.in_port(0)) select_node.out_port(0).data.set_shape(in_node_shape)
def insert_select(graph: Graph, node: Node): context_len = node.frame_time + 1 if context_len == 1: return in_node_port = node.in_port(0).get_source() in_node_shape = node.in_port(0).data.get_shape() node.in_port(0).disconnect() # add Select before saving state to avoid saving garbage select_node = Select(graph, {'name': 'select_' + node.name}).create_node() zero_else = create_const_with_batch_from_input(in_node_port, in_node_shape[1]) select_node.in_port(1).connect(in_node_port) select_node.in_port(2).connect(zero_else.out_port(0)) # check if we have already appropriate iteration counter existing_counters = find_pattern_matches(graph, nodes=[('mem_in', dict(op='ReadValue')), ('mem_in_data', dict(shape=int64_array([context_len]))), ('crop_mem_in', dict(op='Crop', axis=int64_array([1]), offset=int64_array([1]), dim=int64_array([context_len - 1]))), ('crop_mem_in_data', dict()), ('concat', dict(op='Concat', axis=1)), ('concat_data', dict()), ('const_1', dict(op='Const')), ('const_1_data', dict()), ('mem_out', dict(op='Assign')), ('crop_out', dict(op='Crop', axis=int64_array([1]), offset=int64_array([0]), dim=int64_array([1]))), ('crop_out_data', dict()), ('select', dict(op='Select')) ], edges=[('mem_in', 'mem_in_data'), ('mem_in_data', 'crop_mem_in'), ('crop_mem_in', 'crop_mem_in_data'), ('crop_mem_in_data', 'concat', {'in': 0}), ('const_1', 'const_1_data'), ('const_1_data', 'concat', {'in': 1}), ('concat', 'concat_data'), ('concat_data', 'mem_out'), ('concat_data', 'crop_out'), ('crop_out', 'crop_out_data'), ('crop_out_data', 'select')]) counter_match = next(existing_counters, None) if counter_match is not None: ones = Node(graph, inverse_dict(counter_match)['const_1']) input_port = Node(graph, inverse_dict(counter_match)['crop_out']).out_port(0) else: init_value_mem_out = create_const_with_batch_from_input(in_node_port, context_len, precision=np.int32) mem_out = ReadValue(graph, {'name': 'iteration_number', 'variable_id': 'iteration_' + node.name}).create_node() mem_out.in_port(0).connect(init_value_mem_out.out_port(0)) cut_first = Crop(graph, {'name': 'cut_first', 'axis': int64_array([1]), 'offset': int64_array([1]), 'dim': int64_array([context_len - 1])}).create_node() cut_first.in_port(0).connect(mem_out.out_port(0)) ones = create_const_with_batch_from_input(in_node_port, 1, 1, np.int32) concat = Concat(graph, {'name': 'concat_ones', 'in_ports_count': 2, 'axis': 1}).create_node() concat.in_port(0).connect(cut_first.out_port(0)) concat.in_port(1).connect(ones.out_port(0)) mem_in = Assign(graph, {'name': 'iteration_number_out', 'variable_id': 'iteration_' + node.name}).create_node() mem_in.in_port(0).connect(concat.out_port(0)) res = Result(graph, {}).create_node() mem_in.out_port(0).connect(res.in_port(0)) cut_last = Crop(graph, {'name': 'cut_last', 'axis': int64_array([1]), 'offset': int64_array([0]), 'dim': int64_array([1])}).create_node() cut_last.in_port(0).connect(concat.out_port(0)) input_port = cut_last.out_port(0) # Check if data from memory is 1 # if it is True, we have correct data and should proceed with saving it to memory # else we have not gathered context and have garbage here, shouldn't change initial state of memory cast_in = Equal(graph, {'name': input_port.node.name + '/cast_to_bool'}).create_node() cast_in.in_port(0).connect(ones.out_port(0)) cast_in.in_port(1).connect(input_port) select_node.in_port(0).connect(cast_in.out_port(0)) select_node.out_port(0).connect(node.in_port(0)) select_node.out_port(0).data.set_shape(in_node_shape)
def add_output_for_path(graphs_nodes_path): # add output to nodes according to path step_node = graphs_nodes_path[-1]['node'] cur_graph = graphs_nodes_path[-1]['graph'] ports_to_add_nodes = [] for o_p in step_node.out_ports(): ports_to_add_nodes.append(o_p) # update internal_layer_id for new Results for i in range(len(graphs_nodes_path) - 1, 0, -1): cur_max_layer_id = max_internal_layer_id(cur_graph) + 1 cur_loop_node = graphs_nodes_path[i - 1]['node'] new_out_ports = [] if cur_loop_node.op is not 'If': # add Unsqueeze and Result for TensorIterator and Loop and update output_port_map for p_num in ports_to_add_nodes: res_node = add_output_in_body(step_node, p_num, cur_graph, cur_max_layer_id, graphs_nodes_path, i) # IR reader fix output port map for Loop, but have not change for TensorIterator new_port_id = len(cur_loop_node.out_ports()) if cur_loop_node.op == 'TensorIterator': new_port_id = new_port_id + len( cur_loop_node.in_ports()) cur_loop_node.output_port_map.append({ 'axis': 0, 'stride': 1, 'part_size': 1, 'start': 0, 'end': -1, 'external_port_id': new_port_id, 'internal_layer_id': res_node['internal_layer_id'] }) port_id = new_port_id if cur_loop_node.op == 'TensorIterator': port_id = port_id - len(cur_loop_node.in_ports()) new_out_ports.append(port_id) cur_loop_node.add_output_port(port_id) else: # add Result nodes for If and update output_id for p_num in ports_to_add_nodes: res_node = add_output_in_body(step_node, p_num, cur_graph, cur_max_layer_id, graphs_nodes_path, i, add_unsqueeze=False) if cur_loop_node.then_graph == cur_graph: new_port_id = len(cur_loop_node.out_ports()) res_node['output_id'] = new_port_id cur_loop_node.add_output_port(new_port_id) new_out_ports.append(new_port_id) else: res_node['output_id'] = list( cur_loop_node.out_ports().keys())[-1] ports_to_add_nodes = new_out_ports step_node = cur_loop_node cur_graph = graphs_nodes_path[i - 1]['graph'] i = 0 for p_num in ports_to_add_nodes: port = step_node.out_port(p_num) out_name = step_node.soft_get('name', step_node.id) + "." + str(p_num) res_node = Result(cur_graph, {'name': out_name}).create_node() port.connect(res_node.in_port(0)) # add name of Result to fw_tensor_debug_info to avoid renaming if step_node.out_nodes()[p_num].has_and_set( 'fw_tensor_debug_info'): step_node.out_nodes()[p_num]['fw_tensor_debug_info'].append( out_name) else: step_node.out_nodes()[p_num]['fw_tensor_debug_info'] = [[ out_name, out_name ]] if step_node.op == 'TensorIterator': step_node.out_edges()[len(step_node.out_edges())-1]['external_port_id'] = p_num + \ len(step_node.in_ports()) graphs_nodes_path.insert(0, {'node': res_node, 'graph': cur_graph}) i += 1 return graphs_nodes_path
def replace_pattern(graph: Graph, match: dict): node = match['op'] pair_node = Node(graph, node.pair_name) if node.t >= 0: raise Error('Does not support IfDefined with t > 0') if node.in_port(0).get_source() is not None: input_port = node.in_port(0).get_source() op_output_id = node.out_port(0).get_destination().node.id out_port = pair_node.out_port(0) node_name = node.name pair_name = pair_node.name else: input_port = pair_node.in_port(0).get_source() op_output_id = pair_node.out_port(0).get_destination().node.id out_port = node.out_port(0) node_name = pair_node.name pair_name = node.name in_shape = input_port.data.get_shape() node_t = abs(node.t) init_value_memory_out = create_zero_value_with_batch_from_input( input_port, in_shape[1] * node_t) memory_out = ReadValue(graph, { 'name': pair_name, 'variable_id': node_name + pair_name }).create_node() init_value_memory_out.out_port(0).connect(memory_out.in_port(0)) if node_t > 1: crop_concat = Crop( graph, { 'name': 'Memory_crop', 'dim': np.array([in_shape[1] * (node_t - 1)]), 'offset': np.array([in_shape[1]]), 'axis': np.array([1]) }).create_node() memory_out.out_port(0).connect(crop_concat.in_port(0)) concat = Concat(graph, {'name': 'Memory_concat'}).create_node() concat.add_sequence_of_ports('in', range(2)) crop_concat.out_port(0).connect(concat.in_port(0)) concat.in_port(1).connect(input_port) memory_in = Assign(graph, { 'name': node_name, 'variable_id': node_name + pair_name }).create_node() concat.out_port(0).connect(memory_in.in_port(0)) out = Result(graph, {'name': 'Memory_output'}).create_node() memory_in.out_port(0).connect(out.in_port(0)) crop_out = Crop( graph, { 'name': 'Memory_crop_out', 'dim': np.array([in_shape[1]]), 'offset': np.array([0]), 'axis': np.array([1]) }).create_node() memory_out.out_port(0).connect(crop_out.in_port(0)) out_port.get_connection().set_source(crop_out.out_port(0)) else: memory_in = Assign(graph, { 'name': node_name, 'variable_id': node_name + pair_name }).create_node() memory_in.in_port(0).connect(input_port) out = Result(graph, {'name': 'Memory_output'}).create_node() memory_in.out_port(0).connect(out.in_port(0)) out_port.get_connection().set_source(memory_out.out_port(0)) if not graph.graph['cmd_params'].static_shape: log.error( "Model can not be translated in a reshape-able way.\n" "Model Optimizer key static_shape was turned on to prevent related errors.\n" "There will be no success changing input shapes of the model with the help of " "InferenceEngine reshape method", extra={'is_warning': True}) graph.graph['cmd_params'].static_shape = True graph.remove_node(op_output_id) graph.remove_node(node.id) graph.remove_node(pair_node.id)
def replace_pattern(graph: Graph, match: dict): node = match['op'] if node.name == 'iteration_number_out': return # calculate length of context when state of inference becomes meaningful inputs = [] for n in graph.get_op_nodes(**{'op': 'Parameter'}): inputs.append(n) in_nodes = [] for inp in inputs: for ins in inp.out_port(0).get_destinations(): in_nodes.append(ins.node.name) context_len = 1 try: subgraph = invert_sub_graph_between_nodes( graph, [node.in_port(0).get_source().node.name], in_nodes) except Error: return for n in subgraph: n_node = Node(graph, n) if n_node.kind == 'op' and n_node.op == 'Splice': context_len += len(n_node.context) - 1 if context_len == 1: return in_node_port = node.in_port(0).get_source() in_node_shape = node.in_port(0).data.get_shape() node.in_port(0).disconnect() # add Select before saving state to avoid saving garbage select_node = Select(graph, { 'name': 'select_' + node.name }).create_node() zero_else = Const(graph, { 'name': 'zero_else', 'value': np.zeros(in_node_shape) }).create_node() select_node.in_port(1).connect(in_node_port) select_node.in_port(2).connect(zero_else.out_port(0)) # check if we have already appropriate iteration counter existing_counters = find_pattern_matches( graph, nodes=[('mem_in', dict(op='Memory', index=1, shape=int64_array([context_len]))), ('mem_in_data', dict()), ('crop_mem_in', dict(op='Crop', axis=int64_array([1]), offset=int64_array([1]), dim=int64_array([context_len - 1]))), ('crop_mem_in_data', dict()), ('concat', dict(op='Concat', axis=1)), ('concat_data', dict()), ('const_1', dict(op='Const')), ('const_1_data', dict()), ('mem_out', dict(op='Memory', index=0, shape=int64_array([context_len]))), ('crop_out', dict(op='Crop', axis=int64_array([1]), offset=int64_array([0]), dim=int64_array([1]))), ('crop_out_data', dict()), ('select', dict(op='Select'))], edges=[('mem_in', 'mem_in_data'), ('mem_in_data', 'crop_mem_in'), ('crop_mem_in', 'crop_mem_in_data'), ('crop_mem_in_data', 'concat', { 'in': 0 }), ('const_1', 'const_1_data'), ('const_1_data', 'concat', { 'in': 1 }), ('concat', 'concat_data'), ('concat_data', 'mem_out'), ('concat_data', 'crop_out'), ('crop_out', 'crop_out_data'), ('crop_out_data', 'select')]) counter_match = next(existing_counters, None) if counter_match is not None: input_port = Node( graph, inverse_dict(counter_match)['crop_out']).out_port(0) else: mem_out = Memory( graph, { 'name': 'iteration_number', 'size': 2, 'index': 1, 'id': 'iteration_' + node.name, 'shape': int64_array([context_len]), 'dst_type': np.int32 }).create_node() cut_first = Crop( graph, { 'name': 'cut_first', 'axis': int64_array([1]), 'offset': int64_array([1]), 'dim': int64_array([context_len - 1]) }).create_node() cut_first.in_port(0).connect(mem_out.out_port(0)) ones = Const(graph, { 'name': 'ones', 'value': np.ones([1, 1], dtype=np.int32) }).create_node() concat = Concat(graph, { 'name': 'concat_ones', 'in_ports_count': 2, 'axis': 1 }).create_node() concat.in_port(0).connect(cut_first.out_port(0)) concat.in_port(1).connect(ones.out_port(0)) mem_in = Memory( graph, { 'name': 'iteration_number_out', 'size': 2, 'index': 0, 'id': 'iteration_' + node.name, 'shape': int64_array([context_len]) }).create_node() mem_in.in_port(0).connect(concat.out_port(0)) res = Result(graph, {}).create_node() mem_in.out_port(0).connect(res.in_port(0)) cut_last = Crop( graph, { 'name': 'cut_last', 'axis': int64_array([1]), 'offset': int64_array([0]), 'dim': int64_array([1]) }).create_node() cut_last.in_port(0).connect(concat.out_port(0)) input_port = cut_last.out_port(0) select_node.in_port(0).connect(input_port) select_node.out_port(0).connect(node.in_port(0)) select_node.out_port(0).data.set_shape(in_node_shape)
def insert_result(node, child_node, name): res_op = Result(node.graph, { 'name': f'Result_{name}_{node.name}' }).create_node() child_node.out_port(0).connect(res_op.in_port(0))
def replace_op(self, graph: Graph, node: Node): input_out_port = node.in_port(0).get_source() memory_pair_input = unique_id('id') memory_pair_output = unique_id('id') # Input -> FullyConnected fc_layer_after_input_attrs = { 'name': 'input_fullyconnected', 'out-size': node.gifo_x_weights_shape[0], 'transpose_weights': True, 'bias_term': True, } fc_layer_after_input = FullyConnected( graph, fc_layer_after_input_attrs).create_node() fc_layer_after_input.in_port(0).connect(input_out_port) input_as_const(fc_layer_after_input, fc_layer_after_input_attrs, 1, 'weights', node.gifo_x_weights) input_as_const(fc_layer_after_input, fc_layer_after_input_attrs, 2, 'biases', node.gifo_biases) init_value_prev_lstm_output = create_zero_value_with_batch_from_input( input_out_port, node.gifo_r_weights_shape[1]) prev_lstm_output = ReadValue(graph, { 'name': 'prev_memory_output', 'variable_id': memory_pair_input }).create_node() prev_lstm_output.in_port(0).connect( init_value_prev_lstm_output.out_port(0)) # *Memory(output) -> FullyConnected fc_layer_from_prev_state_attrs = { 'name': 'prev_memory_output_fullyconnected', 'out-size': node.gifo_r_weights_shape[0], 'transpose_weights': True, 'bias_term': False, } fc_layer_from_prev_state = FullyConnected( graph, fc_layer_from_prev_state_attrs).create_node() fc_layer_from_prev_state.in_port(0).connect( prev_lstm_output.out_port(0)) input_as_const(fc_layer_from_prev_state, fc_layer_from_prev_state_attrs, 1, 'weights', node.gifo_r_weights) # Memory -> FullyConnected \ # *Eltwise(sum) # Input -> FullyConnected / join_input_prev_state_sum = Add(graph, { 'name': 'join_input_eltwise' }).create_node() join_input_prev_state_sum.in_port(0).connect( fc_layer_from_prev_state.out_port(0)) join_input_prev_state_sum.in_port(1).connect( fc_layer_after_input.out_port(0)) # *Eltwise(sum) -> Split # it is split into 4 nodes: Act, Eltw*3 # the following order is mandatory # ___Tanh # / # Split ---(2)Eltwise(sum) # |\ # | \__(3)Eltwise(sum) # |____(4)Eltwise(sum) split_joined_input_axis = Const(graph, { 'value': np.int64(1) }).create_node() split_joined_input = Split(graph, { 'name': 'join_input_split', 'num_splits': 4, 'out_ports_count': 4 }).create_node() split_joined_input.in_port(0).connect( join_input_prev_state_sum.out_port(0)) split_joined_input.in_port(1).connect( split_joined_input_axis.out_port(0)) # prev_lstm_state = Memory(graph, {'name': 'prev_memory_state', # 'id': memory_pair_output, # 'index': 1, # 'size': 2, # 'shape': np.array([node.input_gate_weights.shape[0]], dtype=np.int64) # }).create_node() init_value_prev_lstm_state = create_zero_value_with_batch_from_input( split_joined_input.out_port(0), node.input_gate_weights.shape[0]) prev_lstm_state = ReadValue(graph, { 'name': 'prev_memory_state', 'variable_id': memory_pair_output }).create_node() prev_lstm_state.in_port(0).connect( init_value_prev_lstm_state.out_port(0)) # *Memory(state) -> *ScaleShift(input) state_input_scaleshift_attrs = { 'name': 'input_scaleshift', 'bias_term': False } state_input_scaleshift = ScaleShiftOp( graph, state_input_scaleshift_attrs).create_node() state_input_scaleshift.in_port(0).connect(prev_lstm_state.out_port(0)) input_as_const(state_input_scaleshift, state_input_scaleshift_attrs, 1, 'weights', node.input_gate_weights) # *Memory(state) -> *ScaleShift(forget) state_forget_scaleshift_attrs = { 'name': 'forget_scaleshift', 'bias_term': False } state_forget_scaleshift = ScaleShiftOp( graph, state_forget_scaleshift_attrs).create_node() state_forget_scaleshift.in_port(0).connect(prev_lstm_state.out_port(0)) input_as_const(state_forget_scaleshift, state_forget_scaleshift_attrs, 1, 'weights', node.forget_gate_weights) # Split \ # (2)Eltwise(sum) # Memory(state) -> *ScaleShift(input) / join_prev_lstm_input_joined_input_sum = Add( graph, { 'name': 'join_prev_lstm_input_joined_input_eltwise' }).create_node() join_prev_lstm_input_joined_input_sum.in_port(0).connect( split_joined_input.out_port(1)) join_prev_lstm_input_joined_input_sum.in_port(1).connect( state_input_scaleshift.out_port(0)) # Split \ # (3)Eltwise(sum) # Memory(state) -> *ScaleShift(forget) / join_prev_lstm_input_joined_forget_sum = Add( graph, { 'name': 'join_prev_lstm_input_joined_forget_sum', }).create_node() join_prev_lstm_input_joined_forget_sum.in_port(0).connect( split_joined_input.out_port(2)) join_prev_lstm_input_joined_forget_sum.in_port(1).connect( state_forget_scaleshift.out_port(0)) # Split -> Tanh remember_tahn = Tanh(graph, {'name': 'remember_tahnv'}).create_node() remember_tahn.in_port(0).connect(split_joined_input.out_port(0)) # Split -> (2)Eltwise(sum) -> *Sigmoid remember_sigmoid = Sigmoid(graph, { 'name': 'remember_sigmoid' }).create_node() remember_sigmoid.in_port(0).connect( join_prev_lstm_input_joined_input_sum.out_port(0)) # Split -> (3)Eltwise(sum) -> **Sigmoid forget_sigmoid = Sigmoid(graph, { 'name': 'forget_sigmoid' }).create_node() forget_sigmoid.in_port(0).connect( join_prev_lstm_input_joined_forget_sum.out_port(0)) # *Memory(state) \ # (6)Eltwise(mul) # Split -> (3)Eltwise(sum) -> **Sigmoid / join_forget_prev_state_mul = Mul(graph, { 'name': 'join_forget_prev_state_mul' }).create_node() join_forget_prev_state_mul.in_port(0).connect( forget_sigmoid.out_port(0)) join_forget_prev_state_mul.in_port(1).connect( prev_lstm_state.out_port(0)) # Split -> Tahn \ # (5)Eltwise(mul) # Split -> (2)Eltwise(sum) -> *Sigmoid / join_remember_candidates_mul = Mul( graph, { 'name': 'join_remember_candidates_mul' }).create_node() join_remember_candidates_mul.in_port(0).connect( remember_tahn.out_port(0)) join_remember_candidates_mul.in_port(1).connect( remember_sigmoid.out_port(0)) # (5)Eltwise(mul) \ # (7)Eltwise(sum) # (6)Eltwise(mul) / join_forget_remember_sum = Add(graph, { 'name': 'join_forget_remember_sum' }).create_node() join_forget_remember_sum.in_port(0).connect( join_forget_prev_state_mul.out_port(0)) join_forget_remember_sum.in_port(1).connect( join_remember_candidates_mul.out_port(0)) # (7)Eltwise(sum) -> Clamp join_forget_clamp = create_op_with_const_inputs( graph, Clamp, { 1: np.array(-node.clip_value, dtype=np.float32), 2: np.array(node.clip_value, dtype=np.float32) }, {'name': 'join_forget_clamp'}, join_forget_remember_sum) # # Clamp -> (2)Memory(state) next_lstm_state = Assign(graph, { 'name': 'next_lstm_state', 'variable_id': memory_pair_output }).create_node() next_lstm_state.in_port(0).connect(join_forget_clamp.out_port(0)) res_node = Result(graph, {'name': 'next_lstm_state_out'}).create_node() res_node.in_port(0).connect(next_lstm_state.out_port(0)) # Clamp -> (2)Tahn state_filtered_tahn = Tanh(graph, { 'name': 'state_filtered_tahn' }).create_node() state_filtered_tahn.in_port(0).connect(join_forget_clamp.out_port(0)) # Clamp -> (2)ScaleShift clamp_scaleshift_attrs = { 'name': 'clamp_scaleshift', 'bias_term': False } clamp_scaleshift = ScaleShiftOp(graph, clamp_scaleshift_attrs).create_node() clamp_scaleshift.in_port(0).connect(join_forget_clamp.out_port(0)) input_as_const(clamp_scaleshift, clamp_scaleshift_attrs, 1, 'weights', node.output_gate_weights) # Split \ # (4)Eltwise(sum) # Clamp -> (2)ScaleShift / join_next_lstm_input_joined_input_sum = Add( graph, { 'name': 'join_next_lstm_input_joined_input_sum', }).create_node() join_next_lstm_input_joined_input_sum.in_port(0).connect( split_joined_input.out_port(3)) join_next_lstm_input_joined_input_sum.in_port(1).connect( clamp_scaleshift.out_port(0)) # (4)Eltwise(sum) -> (3)Sigmoid output_sigmoid = Sigmoid(graph, { 'name': 'output_sigmoid' }).create_node() output_sigmoid.in_port(0).connect( join_next_lstm_input_joined_input_sum.out_port(0)) # (4)Eltwise(sum) -> (3)Sigmoid \ # (5)Eltwise(mul) # Clamp -> (2)Tahn / joined_output_mul = Mul(graph, { 'name': 'joined_output_mul' }).create_node() joined_output_mul.in_port(0).connect(state_filtered_tahn.out_port(0)) joined_output_mul.in_port(1).connect(output_sigmoid.out_port(0)) # (5)Eltwise(mul) -> (3)FullyConnected fc_output_attrs = { 'name': 'FullyConnected', 'out-size': node.projection_weights_shape[0], 'transpose_weights': True, 'bias_term': False } fc_output = FullyConnected(graph, fc_output_attrs).create_node() fc_output.in_port(0).connect(joined_output_mul.out_port(0)) input_as_const(fc_output, fc_output_attrs, 1, 'weights', node.projection_weights) # / (2)Memory(output) # (3)FullyConnected # \ Output (any next node) (edge created automatically after replacement) next_lstm_output = Assign(graph, { 'name': 'next_lstm_output', 'variable_id': memory_pair_input }).create_node() next_lstm_output.in_port(0).connect(fc_output.out_port(0)) res_node_lstm_output = Result(graph, { 'name': 'next_lstm_output_out' }).create_node() res_node_lstm_output.in_port(0).connect(next_lstm_output.out_port(0)) return [fc_output.id]