def replace_sub_graph(self, graph: Graph, match: dict): # This replacer replace ImageScalar operation to Mul->Add sequence # Also it check that weights and biases are good op = match['op'] # Check that weights and biases are not useless has_bias, has_weights = True, True if all([x == 1 for x in np.nditer(op.scale)]): has_weights = False if all([x == 0 for x in np.nditer(op.bias)]): has_bias = False assert len(op.in_ports()) == 1 last_port = op.in_port(0).get_source() # Create Mul & Add nodes if has_weights: mul_weights = Const(graph, dict(value=op.scale, shape=op.scale.shape)).create_node() mul_op = Mul(graph, dict(name=op.id + '/mul_')).create_node() op.in_port(0).get_connection().set_destination(mul_op.in_port(0)) mul_weights.out_port(0).connect(mul_op.in_port(1)) last_port = mul_op.out_port(0) if has_bias: add_bias = Const(graph, dict(value=op.bias, shape=op.bias.shape)).create_node() add_op = Add(graph, dict(name=op.id + '/add_')).create_node() last_port.get_connection().set_destination(add_op.in_port(0)) add_bias.out_port(0).connect(add_op.in_port(1)) last_port = add_op.out_port(0) op.in_port(0).disconnect() op.out_port(0).get_connection().set_source(last_port)
def replace_op(self, graph: Graph, node: Node): if node.module.inverse: axes = Const( graph, { 'value': int64_array(range(2, node.module.num_axes - 1)) }).create_node() dft_node = IDFT(graph, dict(name=node.name, in_ports_count=2)).create_node( [node.in_node(0), axes]) # Slice a real part begin_id = Const(graph, { 'value': int64_array([0, 0]) }).create_node() end_id = Const(graph, {'value': int64_array([0, 1])}).create_node() real = StridedSlice( graph, dict(name=node.name + '/real', begin_mask=[0, 0], end_mask=[0, 1], shrink_axis_mask=[0, 0], new_axis_mask=[0], ellipsis_mask=[1, 0])).create_node( [dft_node, begin_id, end_id]) squeeze_axis = Const(graph, {'value': -1}).create_node() res = Squeeze(graph, dict(name=node.name + '/squeeze')).create_node( [real, squeeze_axis]) return [res.id] else: zero = Const(graph, {'value': 0.0}).create_node() imag = Mul(graph, dict(name=node.name + '/imag')).create_node( [node.in_node(0), zero]) cmplx = PackOp(graph, dict(name=node.name + '/complex', axis=-1)).create_node([node.in_node(0), imag]) axes = Const(graph, { 'value': int64_array(range(2, node.module.num_axes)) }).create_node() dft_node = DFT(graph, dict(name=node.name, in_ports_count=2)).create_node([cmplx, axes]) return [dft_node.id]
def replace_op(self, graph: Graph, node: Node): axis = Const(graph, {'value': int64_array([-1])}).create_node() mvn = MVN( graph, dict(name=node.name + '/mvn', eps=node.module.eps, normalize_variance=True, eps_mode='inside_sqrt')).create_node([node.in_node(0), axis]) weight = node.module.weight.detach().numpy() bias = node.module.bias.detach().numpy() w = Const(graph, {'value': weight}).create_node() b = Const(graph, {'value': bias}).create_node() mul = Mul(graph, dict(name=node.name + '/mul')).create_node([mvn, w]) add = Add(graph, dict(name=node.name + '/add')).create_node([mul, b]) return [add.id]
def replace_pattern(graph: Graph, match: dict): node = match['conv'] # create Reshape before convolution # shape = [in_shape[0], in_shape[1]/patch_stride, 1, patch_stride] shape = Shape(graph, {}).create_node() shape.in_port(0).connect(node.in_port(0).get_source()) split = create_op_with_const_inputs(graph, VariadicSplit, { 1: int64_array(0), 2: int64_array([1, -1]) }, {'out_ports_count': 2}, shape) conv_patch_stride = Const(graph, { 'value': int64_array([node.patch_stride]) }).create_node() pow_node = create_op_node_with_second_input(graph, Pow, int64_array([-1])) pow_node.in_port(0).connect(conv_patch_stride.out_port(0)) mul = Mul(graph, {}).create_node() mul.in_port(0).connect(split.out_port(1)) mul.in_port(1).connect(pow_node.out_port(0)) const_1 = Const(graph, {'value': int64_array([1])}).create_node() concat = Concat(graph, {'in_ports_count': 4, 'axis': 0}).create_node() concat.in_port(0).connect(split.out_port(0)) concat.in_port(1).connect(mul.out_port(0)) concat.in_port(2).connect(const_1.out_port(0)) concat.in_port(3).connect(conv_patch_stride.out_port(0)) reshape_in = Reshape(graph, { 'name': '/Reshape/' + node.name }).create_node() reshape_in.in_port(1).connect(concat.out_port(0)) # create Reshape after Convolution reshape_out = create_op_node_with_second_input( graph, Reshape, int64_array([0, -1]), {'name': node.name + '/Reshape/'}) # connect input_reshape_node source = node.in_port(0).get_source() node.in_port(0).get_connection().set_source(reshape_in.out_port(0)) reshape_in.in_port(0).connect(source) # connect output_reshape_node node.out_port(0).get_connection().set_source(reshape_out.out_port(0)) node.out_port(0).connect(reshape_out.in_port(0))
def replace_interpolate_pattern(graph: Graph, match: dict): split = match['split'] scale = np.array([get_split_scale(split)], dtype=np.float32) axis = int(split.in_port(1).get_connection().get_source().node.value) split_node_name = split.name axis_node = Const(graph, {'name': split_node_name + '/axis', 'value': int64_array([axis])}).create_node() shape_node = Shape(graph, dict(name=split_node_name + '/Shape')).create_node() scales_node = Const(graph, dict(name=split_node_name + '/scales', value=scale)).create_node() mul_node = Mul(graph, dict(name=split_node_name + '/Mul')).create_node() scales_node.out_port(0).connect(mul_node.in_port(1)) strided_slice_node = create_op_with_const_inputs(graph, StridedSlice, {1: int64_array([axis]), 2: int64_array([axis + 1])}, { 'name': split_node_name + '/StridedSlice', 'begin_mask': int64_array([1]), 'end_mask': int64_array([1]), 'new_axis_mask': int64_array([0]), 'shrink_axis_mask': int64_array([0]), 'ellipsis_mask': int64_array([0]) }) shape_node.out_port(0).connect(strided_slice_node.in_port(0)) cast_shape_to_float = Cast(graph, {'dst_type': np.float32}).create_node() strided_slice_node.out_port(0).connect(cast_shape_to_float.in_port(0)) cast_shape_to_float.out_port(0).connect(mul_node.in_port(0)) interp_node = Interpolate(graph, dict(name=split_node_name + '/Interpolate', mode='nearest', antialias=0, pads_begin=int64_array([0]), pads_end=int64_array([0]), coordinate_transformation_mode='half_pixel', nearest_mode='round_prefer_floor', cube_coeff=-0.75, version='opset4', shape_calculation_mode='scales', in_ports_count=4, maybe_part_of_sequence=True)).create_node() floor_node = Floor(graph, {'name': split_node_name + '/Floor'}).create_node() cast_mul_result_to_int = Cast(graph, {'dst_type': np.int64}).create_node() mul_node.out_port(0).connect(floor_node.in_port(0)) floor_node.out_port(0).connect(cast_mul_result_to_int.in_port(0)) cast_mul_result_to_int.out_port(0).connect(interp_node.in_port(1)) scales_node.out_port(0).connect(interp_node.in_port(2)) axis_node.out_port(0).connect(interp_node.in_port(3)) match['concat'].out_port(0).get_connection().set_source(interp_node.out_port(0)) split_connection = split.in_port(0).get_connection() split_connection.set_destination(interp_node.in_port(0)) split_connection.get_source().connect(shape_node.in_port(0))
def find_and_replace_pattern(self, graph: Graph): for node in graph.get_op_nodes(op='ThresholdedRelu'): name = node.soft_get('name', node.id) greater = create_op_with_const_inputs(graph, Greater, {1: float_array([node.alpha])}) greater.in_port(0).connect(node.in_port(0).get_source()) float_greater = Cast(graph, {'dst_type': data_type_str_to_np(graph.graph['cmd_params'].data_type)}).create_node() greater.out_port(0).connect(float_greater.in_port(0)) mul = Mul(graph, {}).create_node() node.out_port(0).get_connection().set_source(mul.out_port(0)) mul.in_port(0).connect(node.in_port(0).get_source()) mul.in_port(1).connect(float_greater.out_port(0)) rename_nodes([(node, name + '/TBR'), (mul, name)])
def replace_op(self, graph: Graph, node: Node): mean = node.module.running_mean.detach().numpy() var = node.module.running_var.detach().numpy() weight = node.module.weight.detach().numpy() bias = node.module.bias.detach().numpy() w = weight / np.sqrt(var + node.module.eps) b = bias - w * mean shape = np.ones(node.module.dims, dtype=np.int32) shape[1] = -1 # channels w = Const(graph, {'value': w.reshape(shape)}).create_node() b = Const(graph, {'value': b.reshape(shape)}).create_node() mul = Mul(graph, dict(name=node.name + '/mul')).create_node( [node.in_node(0), w]) add = Add(graph, dict(name=node.name + '/add')).create_node([mul, b]) return [add.id]
def replace_pattern(self, graph: Graph, match: dict): quantize = match['quantize'] sum_node = Add(graph, dict()).create_node() const = Const(graph, {'value': np.array(0.5)}).create_node() mul_node = Mul(graph, dict()).create_node() mul_node.in_port(0).connect(sum_node.out_port(0)) mul_node.in_port(1).connect(const.out_port(0)) quantize.in_port(1).get_connection().get_source().connect( sum_node.in_port(0)) quantize.in_port(2).get_connection().get_source().connect( sum_node.in_port(1)) quantize.in_port(1).disconnect() quantize.in_port(2).disconnect() mul_node.out_port(0).connect(quantize.in_port(1)) mul_node.out_port(0).connect(quantize.in_port(2))
def replace_op(self, graph: Graph, node: Node): node_name = node.soft_get('name', node.id) rename_node(node, node_name + '/TBR') sqr_node = Mul(graph, {}).create_node() reduce_sum_node = ReduceSum( graph, { 'keep_dims': node.soft_get('keep_dims', 0), 'axis': node.soft_get('axis', None) }).create_node() sqrt_node = create_op_with_const_inputs(graph, Pow, {1: float_array(0.5)}) rename_node(sqrt_node, node_name) # Connect nodes node.in_port(0).get_connection().set_destination(sqr_node.in_port(0)) sqr_node.in_port(0).get_connection().add_destination( sqr_node.in_port(1)) sqr_node.out_port(0).connect(reduce_sum_node.in_port(0)) reduce_sum_node.out_port(0).connect(sqrt_node.in_port(0)) return [sqrt_node.id]
def replace_op(self, graph: Graph, node: Node): # Add new nodes const = Const(graph, dict(value=np.array(-1, dtype=np.int32))).create_node() negate = Mul(graph, {'name': node.name + '/negate_'}).create_node() add = Add(graph, {'name': node.name + '/add_'}).create_node() # Connect nodes node.in_port(1).get_connection().set_destination(negate.in_port(0)) const.out_port(0).connect(negate.in_port(1)) node.in_port(0).get_connection().set_destination(add.in_port(1)) negate.out_port(0).connect(add.in_port(0)) # The "explicit" version of the return value is: [(out_node.id, 0)]) return [add.id]
def replace_interpolate_pattern(graph: Graph, match: dict): split = match['split'] scale = int64_array([get_split_scale(split)]) axis = int(split.in_port(1).get_connection().get_source().node.value) split_node_name = split.name shape_node = Shape(graph, dict(name=split_node_name + '/Shape_')).create_node() scales_node = Const(graph, dict(name=split_node_name + '/scales_', value=scale)).create_node() mul_node = Mul(graph, dict(name=split_node_name + '/Mul_')).create_node() scales_node.out_port(0).connect(mul_node.in_port(1)) slice_begin = Const( graph, dict(name=split_node_name + '/slice_begin_', value=int64_array([axis]))).create_node() slice_end = Const( graph, dict(name=split_node_name + '/slice_end_', value=int64_array([axis + 1]))).create_node() strided_slice_node = StridedSlice( graph, { 'name': split_node_name + '/StridedSlice_', 'begin_mask': int64_array([1]), 'end_mask': int64_array([1]), 'new_axis_mask': int64_array([0]), 'shrink_axis_mask': int64_array([0]), 'ellipsis_mask': int64_array([0]), }).create_node([shape_node, slice_begin, slice_end]) strided_slice_node.out_port(0).connect(mul_node.in_port(0)) interp_node = Interpolate( graph, dict(name=split_node_name + '/Interpolate_', axes=int64_array([axis]), mode='nearest')).create_node() mul_node.out_port(0).connect(interp_node.in_port(1)) match['concat'].out_port(0).get_connection().set_source( interp_node.out_port(0)) split_connection = split.in_port(0).get_connection() split_connection.set_destination(interp_node.in_port(0)) split_connection.get_source().connect(shape_node.in_port(0))
def replace_pattern(graph: Graph, match: [str, Node]): node = match['sub'] # Add new nodes negate_const = Const( graph, dict(name=node.name + '/negate_const', value=np.array(-1))).create_node() negate = Mul(graph, {'name': node.name + '/negate_'}).create_node() add = Add(graph, {'name': node.name + '/add_'}).create_node() # Connect nodes node.in_port(1).get_connection().set_destination(negate.in_port(0)) negate_const.out_port(0).connect(negate.in_port(1)) node.in_port(0).get_connection().set_destination(add.in_port(1)) negate.out_port(0).connect(add.in_port(0)) node.out_port(0).get_connection().set_source(add.out_port(0))
def replace_sub_graph(self, graph: Graph, match: dict): resize_node = match['resize'] if match['mul_1'].in_node(1).value != match['mul_2'].in_node(1).value: log.info('Pattern matched around resize op {} has different scale values.'.format(resize_node.name)) return interpolate_node = Interpolate(graph, {'name': resize_node.name + '/Interpolate', 'mode': resize_node.mode, 'axes': int64_array([2, 3])}).create_node() scale = match['mul_1'].in_node(1).value scale_value = int64_array([scale, scale]) scale_const = Const(graph, {'value': scale_value, 'name': resize_node.name + '/Scale'}).create_node() interpolated_shape = Mul(graph, {'name': resize_node.name + '/OutputShape'}).create_node() match['slice'].out_port(0).connect(interpolated_shape.in_port(0)) scale_const.out_port(0).connect(interpolated_shape.in_port(1)) resize_node.in_port(0).get_connection().set_destination(interpolate_node.in_port(0)) interpolated_shape.out_port(0).connect(interpolate_node.in_port(1)) resize_node.out_port(0).get_connection().set_source(interpolate_node.out_port(0))
def replace_op(self, graph: Graph, node: Node): # Create nodes const_neg = Const( graph, dict(value=np.array(-1), name=node.name + '/negate_const_')).create_node() negate = Mul(graph, {'name': node.name + '/negate_'}).create_node() add = Add(graph, {'name': node.name + '/add_'}).create_node() const = Const(graph, {'value': np.array(2)}).create_node() squared = Pow(graph, {'name': node.name + '/squared_'}).create_node() # Connect nodes node.in_port(0).get_connection().set_destination(add.in_port(0)) node.in_port(1).get_connection().set_destination(negate.in_port(0)) const_neg.out_port(0).connect(negate.in_port(1)) negate.out_port(0).connect(add.in_port(1)) add.out_port(0).connect(squared.in_port(0)) const.out_port(0).connect(squared.in_port(1)) # The "explicit" version of the return value is: [(out_node.id, 0)]) return [squared.id]
def replace_op(self, graph: Graph, node: Node): name = node.soft_get('name', node.id) # create range of axes for MVN based on `start_axis` and rank of input rank = Rank(graph, {'name': name + '/Rank'}).create_node() rng = create_op_with_const_inputs(graph, Range, { 0: int64_array(2), 2: int64_array(1) }, { 'name': name + '/Range', 'output_type': np.int64 }) mvn = MVN( graph, { 'eps': node.epsilon, 'eps_mode': 'inside_sqrt', 'normalize_variance': 1, 'name': name + '/Ins_Norm/MVN_', }).create_node() node.in_port(0).get_connection().set_destination(mvn.in_port(0)) rng.out_port(0).connect(mvn.in_port(1)) mul = Mul(graph, { 'axis': 1, 'name': name + '/Ins_Norm/mul_' }).create_node() mvn.out_port(0).connect(mul.in_port(0)) node.in_port(1).get_connection().set_destination(mul.in_port(1)) add = Add(graph, { 'axis': 1, 'name': name + '/Ins_Norm/add_' }).create_node() mul.out_port(0).connect(add.in_port(0)) node.in_port(2).get_connection().set_destination(add.in_port(1)) mvn.in_port(0).get_connection().add_destination(rank.in_port(0)) rng.in_port(1).connect(rank.out_port(0)) rename_nodes([(node, name + '/TBD'), (add, name)]) return [add.id]
def replace_op(self, graph: Graph, node: Node): # split input to (i_part, f_part, c_part, o_part, ct_1) split_node_axis = Const(graph, {'value': np.int64(1)}).create_node() split_node = Split(graph, { 'name': 'Split_lstm_input_', 'num_splits': 5 }).create_node() node.in_port(0).get_connection().set_destination(split_node.in_port(0)) split_node.in_port(1).connect(split_node_axis.out_port(0)) # i_t = Sigmoid(i_part + w_ic*ct_1) i_scale_attrs = {'name': 'i_scaleshift', 'bias_term': False} i_scale = ScaleShiftOp(graph, i_scale_attrs).create_node() input_as_const(i_scale, i_scale_attrs, 1, 'weights', node.i_weights) split_node.out_port(4).connect(i_scale.in_port(0)) sum_i_c = Add(graph, {'name': 'sum_i_c_'}).create_node() split_node.out_port(0).connect(sum_i_c.in_port(0)) i_scale.out_port(0).connect(sum_i_c.in_port(1)) i_sigmoid = Sigmoid(graph, {'name': 'i_sigmoid'}).create_node() sum_i_c.out_port(0).connect(i_sigmoid.in_port(0)) # f_t = Sigmoid(f_part + w_fc*ct_1) f_scale_attrs = {'name': 'f_scaleshift', 'bias_term': False} f_scale = ScaleShiftOp(graph, f_scale_attrs).create_node() input_as_const(f_scale, f_scale_attrs, 1, 'weights', node.f_weights) split_node.out_port(4).connect(f_scale.in_port(0)) sum_f_c = Add(graph, {'name': 'sum_f_c_'}).create_node() split_node.out_port(1).connect(sum_f_c.in_port(0)) f_scale.out_port(0).connect(sum_f_c.in_port(1)) f_sigmoid = Sigmoid(graph, {'name': 'f_sigmoid'}).create_node() sum_f_c.out_port(0).connect(f_sigmoid.in_port(0)) # c_t = f_t*ct_1 + i_t * tanh(c_part) c_tanh = Tanh(graph, {'name': 'c_tanh'}).create_node() split_node.out_port(2).connect(c_tanh.in_port(0)) prod_i_c_tanh = Mul(graph, {'name': 'prod_i_c_tanh_'}).create_node() i_sigmoid.out_port(0).connect(prod_i_c_tanh.in_port(0)) c_tanh.out_port(0).connect(prod_i_c_tanh.in_port(1)) prod_f_ct_1 = Mul(graph, {'name': 'prod_f_ct_1_'}).create_node() f_sigmoid.out_port(0).connect(prod_f_ct_1.in_port(0)) split_node.out_port(4).connect(prod_f_ct_1.in_port(1)) sum_f_i = Add(graph, {'name': 'sum_f_i_'}).create_node() prod_f_ct_1.out_port(0).connect(sum_f_i.in_port(0)) prod_i_c_tanh.out_port(0).connect(sum_f_i.in_port(1)) # o_t = Sigmoid(o_part + w_oc*c_t) o_scale_attrs = {'name': 'o_scaleshift', 'bias_term': False} o_scale = ScaleShiftOp(graph, o_scale_attrs).create_node() input_as_const(o_scale, o_scale_attrs, 1, 'weights', node.o_weights) sum_f_i.out_port(0).connect(o_scale.in_port(0)) sum_o_c = Add(graph, {'name': 'sum_o_c_'}).create_node() split_node.out_port(3).connect(sum_o_c.in_port(0)) o_scale.out_port(0).connect(sum_o_c.in_port(1)) o_sigmoid = Sigmoid(graph, {'name': 'o_sigmoid'}).create_node() sum_o_c.out_port(0).connect(o_sigmoid.in_port(0)) # m_t = o_t * Tanh(c_t) c_t_tanh = Tanh(graph, {'name': 'c_t_tanh'}).create_node() sum_f_i.out_port(0).connect(c_t_tanh.in_port(0)) prod_o_c_t_tanh = Mul(graph, { 'name': 'prod_o_c_t_tanh_' }).create_node() o_sigmoid.out_port(0).connect(prod_o_c_t_tanh.in_port(0)) c_t_tanh.out_port(0).connect(prod_o_c_t_tanh.in_port(1)) # add concat to create 1 output concat = Concat(graph, {'name': 'Concat_c_m'}).create_node() concat.add_sequence_of_ports('in', range(2)) sum_f_i.out_port(0).connect(concat.in_port(0)) prod_o_c_t_tanh.out_port(0).connect(concat.in_port(1)) return [concat.id]
def replace_pattern(self, graph: Graph, match: Dict[str, Node]): log.debug('UpsampleToResample is triggered') upsample = match['upsample'] input_shape = upsample.in_port(0).data.get_shape() input_shape_rank = len(input_shape) if input_shape_rank not in [4, 5]: log.warning('The input shape is not 4D or 5D for op {}'.format( upsample.soft_get('name'))) return if len(upsample.in_nodes()) == 2: if upsample.in_node(1).value is None: return scales = upsample.in_node(1).value assert scales.shape == (4, ) if not (math.isclose(scales[0], 1, rel_tol=1e-5) and math.isclose(scales[1], 1, rel_tol=1e-5)): return height_scale = scales[2] width_scale = scales[3] else: height_scale = upsample['height_scale'] width_scale = upsample['width_scale'] if 1 in upsample.in_ports() and not upsample.in_port(1).disconnected(): upsample.in_port(1).disconnect() factor = Const(graph, { 'value': np.array([height_scale, width_scale]) }).create_node() shape = Shape(graph, {'name': upsample.name + '/0_port'}).create_node() layout = graph.graph['layout'] if input_shape_rank == 4: begin = Const(graph, { 'value': int64_array([get_height_dim(layout, input_shape_rank)]) }).create_node() else: begin = Const(graph, { 'value': int64_array([get_depth_dim(layout, input_shape_rank)]) }).create_node() end = Const(graph, { 'value': int64_array([get_width_dim(layout, input_shape_rank) + 1]) }).create_node() stride = Const(graph, {'value': int64_array([1])}).create_node() ss = StridedSlice( graph, { 'name': upsample.name + '/ss_0_port', 'begin_mask': np.array([1]), 'end_mask': np.array([0]), 'new_axis_mask': np.array([0]), 'shrink_axis_mask': int64_array([0]), 'ellipsis_mask': int64_array([0]) }).create_node() mul = Mul(graph, { 'name': upsample.name + '/factor_mul_' }).create_node() source = upsample.in_port(0).get_connection().get_source() source.connect(shape.in_port(0)) shape.out_port(0).connect(ss.in_port(0)) begin.out_port(0).connect(ss.in_port(1)) end.out_port(0).connect(ss.in_port(2)) stride.out_port(0).connect(ss.in_port(3)) ss.out_port(0).connect(mul.in_port(0)) factor.out_port(0).connect(mul.in_port(1)) # Create Interpolate operation if input_shape_rank == 4: axes = int64_array([ get_height_dim(layout, input_shape_rank), get_width_dim(layout, input_shape_rank) ]) else: axes = int64_array([ get_depth_dim(layout, input_shape_rank), get_height_dim(layout, input_shape_rank), get_width_dim(layout, input_shape_rank) ]) resample_op = Interpolate( graph, dict(name='Interpolate/{}'.format(upsample.name), axes=axes, mode=upsample.attrs()['mode'], antialias=0, convert_to_resample=True)).create_node() upsample.add_input_port(1, skip_if_exist=True) assert upsample.in_port(1).disconnected() mul.out_port(0).connect(resample_op.in_port(1)) upsample.in_port(0).get_connection().set_destination( resample_op.in_port(0)) upsample.out_port(0).get_connection().set_source( resample_op.out_port(0))
def replace_pattern(self, graph: Graph, match: dict): merge = match['merge'] power = Pow(graph, { 'name': merge.name + '/reciprocal_', 'type': 'PNORM' }).create_node() const1 = Const(graph, { 'value': -1.0, 'name': merge.name + '/negate_const' }).create_node() merge.in_port(0).get_connection().set_destination(power.in_port(0)) const1.out_port(0).connect(power.in_port(1)) concat_node = Concat( graph, { 'axis': 0, 'name': merge.name + '/Concat_', 'override_output_shape': True }).create_node() const3 = Const(graph, { 'name': merge.name + '/const_reduce', 'value': 0 }).create_node() for ii, idx in enumerate( range(merge.significant, merge.to_significant + 1, 1)): const_node = Const( graph, { 'value': float_array(math.pow(10.0, idx)), 'name': merge.name + '/Const_' + ii.__str__() }).create_node() mul_node = Mul(graph, { 'name': merge.name + '/Mul_' + ii.__str__() }).create_node() const_node.out_port(0).connect(mul_node.in_port(0)) power.out_port(0).connect( mul_node.in_port(1)) # connect to the graph node mul_node2 = Mul(graph, { 'name': merge.name + '/Mul_Div_' + ii.__str__() }).create_node() const_node2 = Const( graph, { 'value': float_array(math.pow(10.0, -1 * idx)), 'name': merge.name + '/Const_Pow_' + ii.__str__() }).create_node() cast_node = Cast( graph, { 'name': merge.name + '/Cast_' + idx.__str__(), 'dst_type': np.float32 }).create_node() mul_node.out_port(0).connect(cast_node.in_port(0)) const_node2.out_port(0).connect(mul_node2.in_port(1)) cast_node.out_port(0).connect(mul_node2.in_port(0)) concat_node.add_input_port(ii, skip_if_exist=True) concat_node.in_port(ii).get_connection().set_source( mul_node2.out_port(0)) reducesum_node = ReduceMean( graph, { 'name': merge.id + '/_pnorm_reduced_sum', 'keep_dims': False, 'in_ports_count': 2, 'need_shape_inference': None, 'infer': reduce_infer }).create_node() const3.out_port(0).connect(reducesum_node.in_port(1)) reducesum_node.in_port(0).get_connection().set_source( concat_node.out_port(0)) reshape = Reshape(graph, { 'name': merge.name + '/Reshape_Node' }).create_node() reshape_dim = Const(graph, { 'value': np.array([1, 5]), 'name': merge.id + '/Reshape_Dim' }).create_node() reducesum_node.out_port(0).connect(reshape.in_port(0)) reshape.in_port(1).connect(reshape_dim.out_port(0)) merge.out_port(0).get_connection().set_source(reshape.out_port(0))
def replace_pattern(self, graph: Graph, match: dict): assert match['operator'].has('multiplication_transparent_ports') quantize = match['quantize'] port = match['operator'].input_ports_with(match['quantized']) assert len(port) >= 1 if len(port) > 1: log.debug( 'BinarizeWeightsM1P1 cannot apply transformation for data {} because it consumed more' ' than once'.format(match['quantized'].name)) return assert len(port) == 1 port = port[0] applicable = [ pair for pair in match['operator'].multiplication_transparent_ports if pair[0] == port ] if len(applicable) == 0: return # Look at 3-rd and 4-th inputs of FakeQuantize -- they have constants that should be passed through. # Assume that the constant that should be passed through is a scalar. output_low = quantize.in_node(3) output_high = quantize.in_node(4) assert len(output_low.out_nodes()) == 1 assert len(output_high.out_nodes()) == 1 if not output_low.has_valid('value') and not output_high.has_valid( 'value'): return output_low = output_low.value output_high = output_high.value operator = match['operator'] weights = operator.in_node(1).value weights_rounded = np.round(weights) weights_consistent = np.all(np.isclose(weights, weights_rounded)) and \ set(np.unique(weights_rounded)).issubset({-1, 1}) if weights_consistent and np.all(np.isclose(output_low, 0)) and np.all( np.isclose(output_high, 1)): reduction_indices = set(range(len(weights.shape))) - set( [operator.output_feature_channel]) weights_reduced = np.add.reduce(weights, axis=tuple(reduction_indices)) weights_reduced = weights_reduced.reshape( [len(weights_reduced), 1, 1]) # FIXME: works for NCHW only add_term = Const(graph, {'value': weights_reduced}).create_node() add = Add(graph, {}).create_node() add.in_port(1).connect(add_term.out_port(0)) mul_term = Const(graph, {'value': np.array(0.5)}).create_node() mul = Mul(graph, {}).create_node() mul.in_port(1).connect(mul_term.out_port(0)) add.out_port(0).connect(mul.in_port(0)) operator.out_port(0).get_connection().set_source(mul.out_port(0)) add.in_port(0).connect(operator.out_port(0)) operator['pad_value'] = float(-1.0) elif weights_consistent and np.all(np.isclose( output_low, -1)) and np.all(np.isclose(output_high, +1)): pass else: log.debug( 'ConvToBinaryConv: cannot apply transformation because input range is neither in [0, +1] nor ' 'in [-1, +1].') return operator['type'] = 'BinaryConvolution' operator['mode'] = 'xnor-popcount' operator['pad_value'] = operator.soft_get('pad_value', float(0)) operator['input'] = operator.in_node(0).shape[1] # Weights are not bit-packed yet; there should be a separate transformation to do that assert output_low.size == 1 assert output_high.size == 1 output_low = quantize.in_node(3) output_high = quantize.in_node(4) # Make sure that low/high values are exactly 0/1 output_low.value = np.zeros(output_low.shape) output_high.value = np.ones(output_high.shape)
def parse_specifier(string, graph, layer_node_map): pos = string.find(b'(') if pos == -1: # node name input_name = str(string.split(b' ')[0]).strip('b').replace( "\'", '').replace('\\n', '') if input_name not in layer_node_map: node_name = graph.unique_id(prefix=input_name) graph.add_node(node_name, parameters=[], op="", kind='op') layer_node_map[input_name] = node_name else: node_name = layer_node_map[input_name] return node_name spec = string[:pos] args = get_args_for_specifier(string[pos:]) if spec == b'Append': nodes = [] for i in range(len(args)): nodes.append(parse_specifier(args[i], graph, layer_node_map)) layer_name = 'Append_' for node in nodes: layer_name = layer_name + node + "_" if layer_name not in layer_node_map: concat_name = graph.unique_id(prefix=layer_name) graph.add_node(concat_name, parameters=None, op='concat', kind='op') layer_node_map[layer_name] = concat_name i = 0 Node(graph, concat_name).add_sequence_of_ports('in', range(len(nodes))) for node in nodes: out_port = len(Node(graph, node).out_nodes()) Node(graph, node).add_output_port(out_port) graph.create_edge( Node(graph, node), Node(graph, concat_name), out_port, i, create_edge_attrs(node, concat_name, node, i, out_port)) i = i + 1 else: concat_name = layer_node_map[layer_name] return concat_name elif spec == b'Offset': node = parse_specifier(args[0], graph, layer_node_map) t = int(args[1]) if len(args) > 2: raise Error("ModelOptimizer supports only 2 arguments for Offset") layer_name = 'Offset_' + node + '_' if t < 0: layer_name = layer_name + '_' + str(-t) else: layer_name = layer_name + str(t) if layer_name not in layer_node_map: memory_name = graph.unique_id(prefix=layer_name) layer_node_map[layer_name] = memory_name memory_name_2 = memory_name + '_out' graph.add_node(memory_name, parameters=dict(t=t, pair_name=memory_name_2, has_default=False), op='MemoryOffset', kind='op') out_port = len(Node(graph, node).out_nodes()) in_port = len(Node(graph, memory_name).in_nodes()) Node(graph, memory_name).add_input_port(in_port) Node(graph, node).add_output_port(out_port, skip_if_exist=True) graph.create_edge( Node(graph, node), Node(graph, memory_name), out_port, in_port, create_edge_attrs(node, memory_name, node, in_port, out_port)) else: memory_name = layer_node_map[layer_name] return memory_name elif spec == b'Sum': nodes = [] for i in range(len(args)): nodes.append(parse_specifier(args[i], graph, layer_node_map)) layer_name = 'Sum_' for node in nodes: layer_name = layer_name + node + "_" if layer_name not in layer_node_map: sum_name = graph.unique_id(prefix=layer_name) graph.add_node(sum_name, parameters=None, op='Add', kind='op') layer_node_map[layer_name] = sum_name else: sum_name = layer_node_map[layer_name] for i, node in enumerate(nodes): out_port = len(Node(graph, node).out_nodes()) Node(graph, node).add_output_port(out_port, skip_if_exist=True) Node(graph, sum_name).add_input_port(i) graph.add_edge(node, sum_name, **create_edge_attrs(node, sum_name, node, i)) return sum_name elif spec == b'IfDefined': node_id = parse_specifier(args[0], graph, layer_node_map) node = Node(graph, node_id) if node.op == 'MemoryOffset': node['parameters']['has_default'] = True return node_id elif spec == b'ReplaceIndex': node = parse_specifier(args[0], graph, layer_node_map) return node elif spec == b'Scale': node_name = parse_specifier(args[1], graph, layer_node_map) scale_value = float(args[0]) layer_name = '{}/Mul/{}'.format(node_name, scale_value) if layer_name not in layer_node_map: scale_name = graph.unique_id(prefix=layer_name) scale_node = Mul(graph, {'name': scale_name}).create_node() layer_node_map[layer_name] = scale_name scale_const_name = 'Const_{}'.format(scale_value) const_node = Const(graph, { 'name': scale_const_name, 'value': float_array([scale_value]) }).create_node() node = Node(graph, node_name) graph.create_edge( const_node, scale_node, 0, 0, create_edge_attrs(const_node.id, scale_node.id, const_node.id)) out_port = len(node.out_nodes()) graph.create_edge( node, scale_node, out_port, 1, create_edge_attrs(node_name, scale_node.id, node_name, 1, out_port)) else: scale_name = layer_node_map[layer_name] return scale_name
def _fused_batch_norm_decomposition(graph: Graph, tinput: Port, toutput: Port, gamma: Port, beta: Port, mean: np.ndarray, variance: np.ndarray, can_be_fused=True): """ This is common function for TF, Caffe and MXNet It creates Mul->Add->Mul->Add sub graph """ batch_norm_name = tinput.get_connection().get_destination().node.name # Create first Mul & Add operations mul1_node = Mul( graph, dict(name=batch_norm_name + "/mean", can_be_fused=can_be_fused)).create_node() add1_node = Add( graph, dict(name=batch_norm_name + "/variance", can_be_fused=can_be_fused)).create_node() const_mul1_node = Const(graph, dict(name="data_mul_", value=np.array(mean))).create_node() const_add1_node = Const(graph, dict(name="data_add_", value=np.array(variance))).create_node() # Broadcast const from scalar # We can broadcast only when const.value is scalar if gamma.data.get_shape()[0] != gamma.data.get_value().shape[0]: value = gamma.data.get_value() value.resize(gamma.data.get_shape()).fill(value[0]) gamma.data.set_value(value) # Create second Mul & Add mul2_node = Mul( graph, dict(name=batch_norm_name + "/gamma", can_be_fused=can_be_fused)).create_node() add2_node = Add( graph, dict(name=batch_norm_name + "/beta", can_be_fused=can_be_fused)).create_node() # Connect edges Mul1->Add1->Mul2->Add2 tinput.get_connection().set_destination(mul1_node.in_port(0)) mul1_node.in_port(1).get_connection().set_source( const_mul1_node.out_port(0)) add1_node.in_port(0).get_connection().set_source(mul1_node.out_port(0)) add1_node.in_port(1).get_connection().set_source( const_add1_node.out_port(0)) mul2_node.in_port(0).get_connection().set_source(add1_node.out_port(0)) gamma.get_connection().set_destination(mul2_node.in_port(1)) add2_node.in_port(0).get_connection().set_source(mul2_node.out_port(0)) beta.get_connection().set_destination(add2_node.in_port(1)) toutput.get_connection().set_source(add2_node.out_port(0))
def convert_scale_shift_to_mul_add(graph: Graph): nodes = graph.get_op_nodes(op='ScaleShift') for node in nodes: if node.soft_get('can_be_fused') is False: continue ports_count = len(node.in_ports()) input_port = node.in_port(0) scale_port = node.in_port(1) if ports_count > 1 and not node.in_port( 1).disconnected() else None shift_port = node.in_port(2) if ports_count > 2 and not node.in_port( 2).disconnected() else None output_port = node.out_port(0) has_biases = True has_weights = True # We don't need zero biases if shift_port is None or ( shift_port.data.get_value() is not None and all([x == 0 for x in shift_port.data.get_value()])): has_biases = False # We don't need weights with ones if scale_port is None or ( scale_port.data.get_value() is not None and all([x == 1 for x in scale_port.data.get_value()])): has_weights = False mul_op = Mul(graph, dict(name=node.name + "/Mul_")) add_op = Add(graph, dict(name=node.name + "/Add_")) # Expand dims for current layout broadcast_dims_cnt = len(input_port.data.get_shape( )) - 2 if graph.graph['layout'] == 'NCHW' else 0 # In case if we have constant weights/biases we have to broadcast them according to graph layout # otherwise we insert Reshape with broadcast dim attribute. def broadcast_value(port): value = np.array(port.data.get_value()) for idx in range(broadcast_dims_cnt): value = np.expand_dims(value, axis=-1) port.data.set_value(value) def broadcast_with_reshape(port): input_shape = input_port.data.get_shape() reshape_dims = np.zeros(len(input_shape), dtype=np.int64) for i in range(0, node.axis): reshape_dims[i] = 1 data_shape = port.data.get_shape() for i in range(node.axis, node.axis + len(data_shape)): reshape_dims[i] = data_shape[i - node.axis] for i in range(node.axis + len(data_shape), len(input_shape)): reshape_dims[i] = 1 reshape = create_op_node_with_second_input( graph, Reshape, reshape_dims, dict(name=port.node.name + "/Broadcast_")) port.get_connection().set_destination(reshape.in_port(0)) reshape.out_port(0).connect(port) if has_weights and scale_port.data.get_value() is not None: broadcast_value(scale_port) elif has_weights: broadcast_with_reshape(scale_port) if has_biases and shift_port.data.get_value() is not None: broadcast_value(shift_port) elif has_biases: broadcast_with_reshape(shift_port) if has_biases and has_weights: # Connect input->mul->out->add->out add_node = add_op.create_node() mul_node = mul_op.create_node() # Connect Mul operation with inputs input_port.get_connection().set_destination(mul_node.in_port(0)) scale_port.get_connection().set_destination(mul_node.in_port(1)) # Connect Add operation with inputs mul_node.out_port(0).connect(add_node.in_port(0)) shift_port.get_connection().set_destination(add_node.in_port(1)) output_port.get_connection().set_source(add_node.out_port(0)) elif has_weights: # Connect input->mul->out mul_node = mul_op.create_node() # Connect Mul operation with inputs input_port.get_connection().set_destination(mul_node.in_port(0)) scale_port.get_connection().set_destination(mul_node.in_port(1)) output_port.get_connection().set_source(mul_node.out_port(0)) elif has_biases: # Connect input->add->out add_node = add_op.create_node() # Connect Add operation with inputs input_port.get_connection().set_destination(add_node.in_port(0)) shift_port.get_connection().set_destination(add_node.in_port(1)) output_port.get_connection().set_source(add_node.out_port(0)) else: # Connect input->out producer_port = input_port.get_source() input_port.disconnect() output_port.get_connection().set_source(producer_port)
def replace_pattern(graph: Graph, match: dict): log.debug( '================== GNMTBeforeConditionFind ==================') input_sequence_lengths = match['Max'].in_port(0).get_source().node encoder_sequence_lengths = looking_for_op_in_list([ port.node for port in input_sequence_lengths.out_port(0).get_destinations() ], 'Identity') # Looking for Sequence_length node in encoder looks like: # Sequence_length -> CheckSeqLen -> Max -> Maximum -> Minimum check_seq_len = looking_for_op_in_list([ port.node for port in encoder_sequence_lengths.out_port( 0).get_destinations() ], 'Identity') max = looking_for_op_in_list([ port.node for port in check_seq_len.out_port(0).get_destinations() ], 'ReduceMax') maximum = max.out_port(0).get_destinations()[0].node assert maximum.op == 'Maximum' minimum = maximum.out_port(0).get_destinations()[0].node assert minimum.op == 'Minimum' tensor_seq_len = looking_for_op_in_list([ minimum.in_port(port).get_source().node for port in minimum.in_ports() ], 'StridedSlice') # Create node for multiplying seq_len by 2 const = Const(graph, { 'name': 'FakeSeqLenMultiplyer', 'value': np.array(2) }).create_node() mul_op = Mul(graph, {'name': 'FakeSeqLen'}).create_node() const.out_port(0).get_connection().set_destination(mul_op.in_port(1)) tensor_seq_len.out_port(0).get_connection().add_destination( mul_op.in_port(0)) # Connect seq_len * 2 to TensorArray from GNMT loop ta_writes = [ port.node for port in match['Identity_1'].out_port(0).get_destinations() if port.node.op == 'TensorArrayWriteV3' ] for ta_write in ta_writes: ta = ta_write.in_port(0).get_source().node.in_port( 0).get_source().node ta.in_port(0).disconnect() ta.in_port(0).get_connection().set_source(mul_op.out_port(0)) if not graph.graph['cmd_params'].static_shape: log.error( "Model can not be translated in a reshape-able way.\n" "Model Optimizer key static_shape was turned on to prevent related errors.\n" "There will be no success changing input shapes of the model with the help of " "InferenceEngine reshape method", extra={'is_warning': True}) graph.graph['cmd_params'].static_shape = True
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch): reshape_classes_node = create_op_node_with_second_input( graph, Reshape, int64_array([0, -1]), dict(name='do_reshape_classes'), match.single_input_node(1)[0]) priors_node = match.single_input_node(2)[0] placeholder = [ Node(graph, node_id) for node_id in graph.nodes() if Node(graph, node_id).op == 'Parameter' ][0] im_height = placeholder.shape[1] im_width = placeholder.shape[2] # scale prior boxes to the [0, 1] interval priors_scale_const_node = Const( graph, { 'value': np.array( [1 / im_width, 1 / im_height, 1 / im_width, 1 / im_height]) }).create_node([]) priors_scale_node = Mul(graph, { 'name': 'scale_priors' }).create_node([priors_node, priors_scale_const_node]) # calculate prior boxes widths and heights split_node = SplitV(graph, { 'axis': 2, 'size_splits': [1, 1, 1, 1], 'out_ports_count': 4 }).create_node([priors_scale_node]) priors_width_node = Sub( graph, dict(name=split_node.name + '/sub_2-0_')).create_node([ (split_node, 2), (split_node, 0) ]) priors_height_node = Sub(graph, dict(name=split_node.name + '/sub_3-1_')).create_node([ (split_node, 3), (split_node, 1) ]) # concat weights and heights into a single tensor and multiple with the box coordinates regression values concat_width_height_node = Concat(graph, { 'name': 'concat_priors_width_height', 'axis': -1, 'in_ports_count': 4 }).create_node([ priors_width_node, priors_height_node, priors_width_node, priors_height_node ]) applied_width_height_regressions_node = Mul(graph, { 'name': 'final_regressions' }).create_node( [concat_width_height_node, match.single_input_node(0)[0]]) # reshape to 2D tensor as Inference Engine Detection Output layer expects reshape_regression_node = create_op_node_with_second_input( graph, Reshape, int64_array([0, -1]), dict(name='reshape_regression'), applied_width_height_regressions_node) detection_output_op = DetectionOutput( graph, match.custom_replacement_desc.custom_attributes) detection_output_op.attrs['old_infer'] = detection_output_op.attrs[ 'infer'] detection_output_op.attrs['infer'] = __class__.do_infer detection_output_node = detection_output_op.create_node( [reshape_regression_node, reshape_classes_node, priors_scale_node], dict(name=detection_output_op.attrs['type'], clip=1, normalized=1, variance_encoded_in_target=0)) return {'detection_output_node': detection_output_node}
def replace_resize(graph: Graph, resize: Node): log.debug("Converting of ONNX Resize-11 to Interpolate-4 " "is triggered for node {}.".format( resize.soft_get('name', resize.id))) input_shape = resize.in_port(0).data.get_shape() input_rank = len(input_shape) resize_name = resize.soft_get('name', resize.id) if input_rank not in {4, 5}: log.warning( 'The input shape is not 4D or 5D for op with name {}'.format( resize_name)) return num_of_inputs = len([ port for port in resize.in_ports().values() if not port.disconnected() ]) assert num_of_inputs in {3, 4}, \ "Number of inputs of ONNXResize (with name {}) should be equal to 3 or 4".format(resize_name) assert resize.soft_get('coordinate_transformation_mode') != 'tf_crop_and_resize', \ 'Mode tf_crop_and_resize is not supported for op {} with name {}'.format(resize.op, resize_name) layout = graph.graph['layout'] if input_rank == 4: begin_dim = get_height_dim(layout, input_rank) end_dim = get_width_dim(layout, input_rank) + 1 else: begin_dim = get_depth_dim(layout, input_rank) end_dim = get_width_dim(layout, input_rank) + 1 sizes_ss = create_op_with_const_inputs( graph, StridedSlice, { 1: int64_array([begin_dim]), 2: int64_array([end_dim]), 3: int64_array([1]) }, { 'name': resize_name + '/StridedSlice_sizes', 'begin_mask': int64_array([1]), 'end_mask': int64_array([1]), 'new_axis_mask': int64_array([0]), 'shrink_axis_mask': int64_array([0]), 'ellipsis_mask': int64_array([0]) }) scales_ss = create_op_with_const_inputs( graph, StridedSlice, { 1: int64_array([begin_dim]), 2: int64_array([end_dim]), 3: int64_array([1]) }, { 'name': resize_name + '/StridedSlice_scales', 'begin_mask': int64_array([1]), 'end_mask': int64_array([1]), 'new_axis_mask': int64_array([0]), 'shrink_axis_mask': int64_array([0]), 'ellipsis_mask': int64_array([0]) }) axes_node = Const( graph, { 'name': resize_name + '/axis', 'value': int64_array(np.arange(begin_dim, end_dim)) }).create_node() shape_calculation_mode = 'scales' if num_of_inputs == 3 else 'sizes' interpolate_node = Interpolate( graph, { 'version': 'opset4', 'mode': convert_mode(resize.mode), 'coordinate_transformation_mode': resize.coordinate_transformation_mode, 'cube_coeff': resize.cube_coeff, 'nearest_mode': resize.nearest_mode, 'pads_begin': int64_array([0]), 'pads_end': int64_array([0]), 'antialias': 0, 'shape_calculation_mode': shape_calculation_mode, 'in_ports_count': 4 }).create_node() axes_node.out_port(0).connect(interpolate_node.in_port(3)) shape_of = Shape(graph, {'name': resize_name + '/ShapeOf'}).create_node() add_node = create_op_with_const_inputs(graph, Add, {1: float_array([1.0e-5])}, {'name': resize_name + '/Add'}) input_data_type = data_type_str_to_np(graph.graph['cmd_params'].data_type) if num_of_inputs == 3: cast_shape_to_float = Cast(graph, { 'dst_type': input_data_type }).create_node() mul_node = Mul(graph, {'name': resize_name + '/Mul'}).create_node() shape_of.out_port(0).connect(cast_shape_to_float.in_port(0)) cast_shape_to_float.out_port(0).connect(mul_node.in_port(0)) cast_add_result_to_int = Cast(graph, { 'dst_type': np.int64 }).create_node() floor_node = Floor(graph, { 'name': resize_name + '/Floor' }).create_node() mul_node.out_port(0).connect(add_node.in_port(0)) add_node.out_port(0).connect(floor_node.in_port(0)) floor_node.out_port(0).connect(cast_add_result_to_int.in_port(0)) cast_add_result_to_int.out_port(0).connect(sizes_ss.in_port(0)) sizes_ss.out_port(0).connect(interpolate_node.in_port(1)) scales_ss.out_port(0).connect(interpolate_node.in_port(2)) connection_of_resize_input = resize.in_port(0).get_connection() connection_of_resize_input.set_destination(interpolate_node.in_port(0)) connection_of_scales = resize.in_port(2).get_connection() connection_of_scales.set_destination(scales_ss.in_port(0)) connection_of_resize_input.get_source().connect(shape_of.in_port(0)) connection_of_scales.get_source().connect(mul_node.in_port(1)) else: cast_shape_to_float = Cast(graph, { 'dst_type': input_data_type }).create_node() cast_sizes_to_float = Cast(graph, { 'dst_type': input_data_type }).create_node() div_node = Div(graph, {'name': resize_name + '/Div'}).create_node() cast_sizes_to_float.out_port(0).connect(div_node.in_port(0)) cast_shape_to_float.out_port(0).connect(div_node.in_port(1)) shape_of.out_port(0).connect(cast_shape_to_float.in_port(0)) div_node.out_port(0).connect(add_node.in_port(0)) add_node.out_port(0).connect(scales_ss.in_port(0)) scales_ss.out_port(0).connect(interpolate_node.in_port(2)) sizes_ss.out_port(0).connect(interpolate_node.in_port(1)) connection_of_resize_input = resize.in_port(0).get_connection() connection_of_resize_input.set_destination(interpolate_node.in_port(0)) connection_of_sizes = resize.in_port(3).get_connection() connection_of_sizes.set_destination(sizes_ss.in_port(0)) connection_of_resize_input.get_source().connect(shape_of.in_port(0)) connection_of_sizes.get_source().connect( cast_sizes_to_float.in_port(0)) rename_nodes([(resize, resize_name + '/delete'), (interpolate_node, resize_name)]) resize.out_port(0).get_connection().set_source( interpolate_node.out_port(0))
def replace_pattern(graph: Graph, match: dict): node = match['conv'] node_name = node.soft_get('name', node.id) # create Reshape before convolution # shape = [in_shape[0], in_shape[1]/patch_stride, 1, patch_stride] shape = Shape(graph, {'name': node_name + '/Shape'}).create_node() shape.in_port(0).connect(node.in_port(0).get_source()) split = create_op_with_const_inputs(graph, VariadicSplit, { 1: int64_array(0), 2: int64_array([1, -1]) }, { 'name': shape.name + '/split_batch', 'out_ports_count': 2 }, shape) pow_node = create_op_node_with_second_input( graph, Pow, int64_array([-1]), {'name': node_name + '/patch_stride/inverse'}) conv_patch_stride = Const( graph, { 'value': int64_array([node.patch_stride]), 'name': node_name + '/patch_stride/' }).create_node() pow_node.in_port(0).connect(conv_patch_stride.out_port(0)) mul = Mul(graph, { 'name': node_name + '/mul_inverse_stride_h' }).create_node() mul.in_port(0).connect(split.out_port(1)) mul.in_port(1).connect(pow_node.out_port(0)) concat = create_op_with_const_inputs( graph, Concat, {2: int64_array([1])}, { 'name': node_name + '/concat_all_dims', 'in_ports_count': 4, 'axis': 0 }) concat.in_port(0).connect(split.out_port(0)) concat.in_port(1).connect(mul.out_port(0)) concat.in_port(3).connect(conv_patch_stride.out_port(0)) reshape_in = Reshape(graph, { 'name': node_name + '/reshape_in' }).create_node() reshape_in.in_port(1).connect(concat.out_port(0)) # create Reshape after Convolution reshape_out = create_op_node_with_second_input( graph, Reshape, int64_array([0, -1]), {'name': node_name + '/reshape_out'}) # connect input_reshape_node source = node.in_port(0).get_source() node.in_port(0).get_connection().set_source(reshape_in.out_port(0)) reshape_in.in_port(0).connect(source) # connect output_reshape_node node.out_port(0).get_connection().set_source(reshape_out.out_port(0)) node.out_port(0).connect(reshape_out.in_port(0))
def _fuse_linear_sequence(graph: Graph, start_node: Node): """ This function finds the sequence of Mul/Add operations and replaces this sequence with two ops (Mul->Add). :param graph: :param start_node: The first operation of the sequence """ fnodes = [start_node] while True: node = fnodes[-1] destinations = node.out_port(0).get_destinations() if len(destinations) != 1: break dst_node = destinations[0].node if dst_node.soft_get('op') in [ 'Mul', 'Add' ] and get_value_in_port(dst_node) is not None and dst_node.soft_get( 'can_be_fused') is True: fnodes.append(dst_node) else: break if len(fnodes) == 1 or (len(fnodes) == 2 and fnodes[0].op == 'Mul' and fnodes[1].op == 'Add'): return False input_shape = get_tensor_in_port(start_node).data.get_shape() init_dims_cnt = len( input_shape) - 2 if graph.graph['layout'] == 'NCHW' else 1 mul = np.ones([1 for x in range(init_dims_cnt)]) add = np.zeros([1 for x in range(init_dims_cnt)]) first_mul_name = None first_add_name = None for node in fnodes: const_port_value = get_value_in_port(node).data.get_value() if node.op == 'Mul': if first_mul_name is None: first_mul_name = node.name mul = mul * const_port_value add = add * const_port_value elif node.op == 'Add': if first_add_name is None: first_add_name = node.name add = add + const_port_value # If mul is scalar we broadcast it to biases shape if mul.shape != add.shape and len(mul.shape) == 1 and mul.shape[0] == 1: mul = np.array([mul[0] for x in range(add.shape[0])]) assert (np.array_equal( get_tensor_in_port(fnodes[0]).data.get_shape(), fnodes[-1].out_port(0).data.get_shape())) mul_op = Mul(graph, dict(name='{}/Fused_Mul_'.format(first_mul_name or ''))) add_op = Add(graph, dict(name='{}/Fused_Add_'.format(first_add_name or ''))) in_port = get_tensor_in_port(fnodes[0]) out_port = fnodes[-1].out_port(0) """ Four cases considered below: 1. Mul and Add have valid values (mul value != 1 and add value != 0) 2. Only Mul has valid values, so we add only Mul node 3. Only Add has valid values, so we add only Add node 4. When Mul and Add has not valid values we just merge two data nodes """ if any([x != 0 for x in np.nditer(add)]) and any([x != 1 for x in np.nditer(mul)]): # Const\ Const\ # ----->Mul------>Add--> mul_const = Const(graph, dict(name="data_mul_", value=np.array(mul))).create_node() add_const = Const(graph, dict(name="data_add_", value=np.array(add))).create_node() mul_node = mul_op.create_node() add_node = add_op.create_node() in_port.get_connection().set_destination(mul_node.in_port(0)) mul_const.out_port(0).connect(mul_node.in_port(1)) mul_node.out_port(0).connect(add_node.in_port(0)) add_const.out_port(0).connect(add_node.in_port(1)) out_port.get_connection().set_source(add_node.out_port(0)) elif any([x != 1 for x in np.nditer(mul)]): # Const\ # ----->Mul--> mul_const = Const(graph, dict(name="data_mul_", value=np.array(mul))).create_node() mul_node = mul_op.create_node() in_port.get_connection().set_destination(mul_node.in_port(0)) mul_const.out_port(0).connect(mul_node.in_port(1)) out_port.get_connection().set_source(mul_node.out_port(0)) elif any([x != 0 for x in np.nditer(add)]): # Const\ # ----->Add--> add_const = Const(graph, dict(name="data_add_", value=np.array(add))).create_node() add_node = add_op.create_node() in_port.get_connection().set_destination(add_node.in_port(0)) add_const.out_port(0).connect(add_node.in_port(1)) out_port.get_connection().set_source(add_node.out_port(0)) else: source_node = in_port.get_source() in_port.disconnect() out_port.get_connection().set_source(source_node) # Remove fused nodes for node in fnodes: graph.remove_node(node.id) log.debug('Fused {} operations'.format(len(fnodes))) return True
def replace_op(self, graph: Graph, node: Node): input_node = node.in_node() memory_pair_input = unique_id('id') memory_pair_output = unique_id('id') # Input -> FullyConnected fc_layer_after_input_attrs = { 'name': 'input_fullyconnected', 'num_output': node.gifo_x_weights_shape[0], 'bias_term': True } embed_input(fc_layer_after_input_attrs, 1, 'weights', node.gifo_x_weights) embed_input(fc_layer_after_input_attrs, 2, 'biases', node.gifo_biases) fc_layer_after_input = InnerProduct( graph, fc_layer_after_input_attrs).create_node([input_node]) prev_lstm_output = Memory( graph, { 'name': 'prev_memory_output', 'id': memory_pair_input, 'index': 1, 'size': 2, 'shape': np.array([node.gifo_r_weights_shape[1]], dtype=np.int64) }).create_node() # *Memory(output) -> FullyConnected fc_layer_from_prev_state_attrs = { 'name': 'prev_memory_output_fullyconnected', 'num_output': node.gifo_r_weights_shape[0], 'bias_term': False } embed_input(fc_layer_from_prev_state_attrs, 1, 'weights', node.gifo_r_weights) fc_layer_from_prev_state = InnerProduct( graph, fc_layer_from_prev_state_attrs).create_node([prev_lstm_output]) # Memory -> FullyConnected \ # *Eltwise(sum) # Input -> FullyConnected / join_input_prev_state_sum = Add(graph, { 'name': 'join_input_eltwise', }).create_node([fc_layer_from_prev_state, fc_layer_after_input]) # *Eltwise(sum) -> Split # it is split into 4 nodes: Act, Eltw*3 # the following order is mandatory # ___Tanh # / # Split ---(2)Eltwise(sum) # |\ # | \__(3)Eltwise(sum) # |____(4)Eltwise(sum) split_joined_input = Split( graph, { 'name': 'join_input_split', 'axis': 1, 'num_split': 4, 'out_ports_count': 4, }).create_node([join_input_prev_state_sum]) prev_lstm_state = Memory( graph, { 'name': 'prev_memory_state', 'id': memory_pair_output, 'index': 1, 'size': 2, 'shape': np.array([node.input_gate_weights.shape[0]], dtype=np.int64) }).create_node() # *Memory(state) -> *ScaleShift(input) state_input_scaleshift_attrs = { 'name': 'input_scaleshift', 'bias_term': False } embed_input(state_input_scaleshift_attrs, 1, 'weights', node.input_gate_weights) state_input_scaleshift = ScaleShiftOp( graph, state_input_scaleshift_attrs).create_node([prev_lstm_state]) # *Memory(state) -> *ScaleShift(forget) state_forget_scaleshift_attrs = { 'name': 'forget_scaleshift', 'bias_term': False } embed_input(state_forget_scaleshift_attrs, 1, 'weights', node.forget_gate_weights) state_forget_scaleshift = ScaleShiftOp( graph, state_forget_scaleshift_attrs).create_node([prev_lstm_state]) # Split \ # (2)Eltwise(sum) # Memory(state) -> *ScaleShift(input) / join_prev_lstm_input_joined_input_sum = Add( graph, { 'name': 'join_prev_lstm_input_joined_input_eltwise', }).create_node([(split_joined_input, 1), state_input_scaleshift]) # Split \ # (3)Eltwise(sum) # Memory(state) -> *ScaleShift(forget) / join_prev_lstm_input_joined_forget_sum = Add( graph, { 'name': 'join_prev_lstm_input_joined_forget_sum', }).create_node([(split_joined_input, 2), state_forget_scaleshift]) # Split -> Tanh remember_tahn = Tanh(graph, { 'name': 'remember_tahnv' }).create_node([(split_joined_input, 0)]) # Split -> (2)Eltwise(sum) -> *Sigmoid remember_sigmoid = Sigmoid(graph, { 'name': 'remember_sigmoid' }).create_node([join_prev_lstm_input_joined_input_sum]) # Split -> (3)Eltwise(sum) -> **Sigmoid forget_sigmoid = Sigmoid(graph, { 'name': 'forget_sigmoid' }).create_node([join_prev_lstm_input_joined_forget_sum]) # *Memory(state) \ # (6)Eltwise(mul) # Split -> (3)Eltwise(sum) -> **Sigmoid / join_forget_prev_state_mul = Mul(graph, { 'name': 'join_forget_prev_state_mul', }).create_node([forget_sigmoid, prev_lstm_state]) # Split -> Tahn \ # (5)Eltwise(mul) # Split -> (2)Eltwise(sum) -> *Sigmoid / join_remember_candidates_mul = Mul( graph, { 'name': 'join_remember_candidates_mul', }).create_node([remember_tahn, remember_sigmoid]) # (5)Eltwise(mul) \ # (7)Eltwise(sum) # (6)Eltwise(mul) / join_forget_remember_sum = Add(graph, { 'name': 'join_forget_remember_sum', }).create_node( [join_forget_prev_state_mul, join_remember_candidates_mul]) # (7)Eltwise(sum) -> Clamp join_forget_clamp = Clamp( graph, { 'name': 'join_forget_clamp', 'max': node.clip_value, 'min': -node.clip_value }).create_node([join_forget_remember_sum]) # # Clamp -> (2)Memory(state) next_lstm_state = Memory( graph, { 'name': 'next_lstm_state', 'id': memory_pair_output, 'index': 0, 'size': 2, 'shape': np.array([node.input_gate_weights.shape[0]], dtype=np.int64) }).create_node([join_forget_clamp]) Result(graph, { 'name': 'next_lstm_state_out' }).create_node([next_lstm_state]) # Clamp -> (2)Tahn state_filtered_tahn = Tanh(graph, { 'name': 'state_filtered_tahn' }).create_node([join_forget_clamp]) # Clamp -> (2)ScaleShift clamp_scaleshift_attrs = { 'name': 'clamp_scaleshift', 'bias_term': False } embed_input(clamp_scaleshift_attrs, 1, 'weights', node.output_gate_weights) clamp_scaleshift = ScaleShiftOp( graph, clamp_scaleshift_attrs).create_node([join_forget_clamp]) # Split \ # (4)Eltwise(sum) # Clamp -> (2)ScaleShift / join_next_lstm_input_joined_input_sum = Add( graph, { 'name': 'join_next_lstm_input_joined_input_sum', }).create_node([(split_joined_input, 3), clamp_scaleshift]) # (4)Eltwise(sum) -> (3)Sigmoid output_sigmoid = Sigmoid(graph, { 'name': 'output_sigmoid' }).create_node([join_next_lstm_input_joined_input_sum]) # (4)Eltwise(sum) -> (3)Sigmoid \ # (5)Eltwise(mul) # Clamp -> (2)Tahn / joined_output_mul = Mul(graph, { 'name': 'joined_output_mul' }).create_node([state_filtered_tahn, output_sigmoid]) # (5)Eltwise(mul) -> (3)FullyConnected fc_output_attrs = { 'name': 'FullyConnected', 'num_output': node.projection_weights_shape[0], 'bias_term': False } embed_input(fc_output_attrs, 1, 'weights', node.projection_weights) fc_output = InnerProduct(graph, fc_output_attrs).create_node( [joined_output_mul]) # / (2)Memory(output) # (3)FullyConnected # \ Output (any next node) (edge created automatically after replacement) next_lstm_output = Memory( graph, { 'name': 'next_lstm_output', 'id': memory_pair_input, 'index': 0, 'size': 2, 'shape': np.array([node.gifo_r_weights_shape[1]], dtype=np.int64) }).create_node([fc_output]) Result(graph, { 'name': 'next_lstm_output_out' }).create_node([next_lstm_output]) return [fc_output.id]
def extract(cls, node): Mul.update_node_stat( node, {'data_type': tf_dtype_extractor(node.pb.attr["T"].type)}) return cls.enabled
def replace_sub_graph(self, graph: Graph, match: Dict[str, Node]): node = match['op'] name = node.name min_port_tuple = (node.in_port(1).get_source().node, node.in_port(1).get_source().idx) max_port_tuple = (node.in_port(2).get_source().node, node.in_port(2).get_source().idx) node.in_port(1).disconnect() node.in_port(2).disconnect() # make sure min < max min_less_max = Less(graph, { 'name': name + '/if_min_less_max' }).create_node([min_port_tuple, max_port_tuple]) minimum = Select(graph, { 'name': name + '/minimum' }).create_node([min_less_max, min_port_tuple, max_port_tuple]) maximum = Select(graph, { 'name': name + '/maximum' }).create_node([min_less_max, max_port_tuple, min_port_tuple]) # to create zero of limits data type, we multiply it by integer zero zero = create_op_node_with_second_input(graph, Mul, int64_array(0), {'name': name + '/zero'}, input_node=minimum) # if 0 < min < max: min_adj = 0 and max_adj = max - min min_greater_zero = Greater(graph, { 'name': name + '/if_minimum_greater_zero' }).create_node([minimum, zero]) max_minus_min = Sub(graph, { 'name': name + '/max_minus_min' }).create_node([maximum, minimum]) minimum = Select(graph, { 'name': name + '/first_adj_min' }).create_node([min_greater_zero, zero, minimum]) maximum = Select(graph, { 'name': name + '/first_adj_max' }).create_node([min_greater_zero, max_minus_min, maximum]) # if min < max < 0: min_adj = min - max and max_adj = 0 max_less_zero = Less(graph, { 'name': name + '/if_max_less_zero' }).create_node([maximum, zero]) min_minus_max = Sub(graph, { 'name': name + '/min_minus_max' }).create_node([minimum, maximum]) minimum = Select(graph, { 'name': name + '/second_adj_min' }).create_node([max_less_zero, min_minus_max, minimum]) maximum = Select(graph, { 'name': name + '/second_adj_max' }).create_node([max_less_zero, zero, maximum]) # scale = (max - min) / (2 ^ num_bits - 1), float_range = Sub(graph, { 'name': name + '/float_range' }).create_node([maximum, minimum]) quant_min_value, quant_max_value = int( node.narrow_range), 2**node.num_bits - 1 int_range = Const( graph, dict(name=name + '/int_range', value=quant_max_value - quant_min_value)).create_node() scale = Div(graph, { 'name': name + '/scale' }).create_node([float_range, int_range]) # min_adj = scale * round(min / scale) descaled_min = Div(graph, { 'name': name + '/descaled_min' }).create_node([minimum, scale]) rounded_descaled_min = Round(graph, { 'name': name + '/rounded_descaled_min' }).create_node([descaled_min]) min_adj = Mul(graph, { 'name': name + '/min_adj' }).create_node([scale, rounded_descaled_min]) # max_adj = max + min_adj - min. adjustment = Sub(graph, { 'name': name + '/limits_adjustment' }).create_node([min_adj, minimum]) max_adj = Add(graph, { 'name': name + '/max_adj' }).create_node([maximum, adjustment]) # FakeQuantize operation has 5 inputs instead of 3 inputs in TensorFlow node.add_input_port(3, skip_if_exist=True) node.add_input_port(4, skip_if_exist=True) node.in_port(1).connect(min_adj.out_port(0)) node.in_port(2).connect(max_adj.out_port(0)) node.in_port(3).connect(min_adj.out_port(0)) node.in_port(4).connect(max_adj.out_port(0)) FakeQuantize.update_node_stat(node, {'levels': node['levels']})