def replace_sub_graph(self, graph: Graph, match: dict):
# This replacer replace ImageScalar operation to Mul->Add sequence
# Also it check that weights and biases are good
op = match['op']
# Check that weights and biases are not useless
has_bias, has_weights = True, True
if all([x == 1 for x in np.nditer(op.scale)]):
has_weights = False
if all([x == 0 for x in np.nditer(op.bias)]):
has_bias = False
assert len(op.in_ports()) == 1
last_port = op.in_port(0).get_source()
# Create Mul & Add nodes
if has_weights:
mul_weights = Const(graph,
dict(value=op.scale,
shape=op.scale.shape)).create_node()
mul_op = Mul(graph, dict(name=op.id + '/mul_')).create_node()
op.in_port(0).get_connection().set_destination(mul_op.in_port(0))
mul_weights.out_port(0).connect(mul_op.in_port(1))
last_port = mul_op.out_port(0)
if has_bias:
add_bias = Const(graph, dict(value=op.bias,
shape=op.bias.shape)).create_node()
add_op = Add(graph, dict(name=op.id + '/add_')).create_node()
last_port.get_connection().set_destination(add_op.in_port(0))
add_bias.out_port(0).connect(add_op.in_port(1))
last_port = add_op.out_port(0)
op.in_port(0).disconnect()
op.out_port(0).get_connection().set_source(last_port)
def replace_op(self, graph: Graph, node: Node):
if node.module.inverse:
axes = Const(
graph, {
'value': int64_array(range(2, node.module.num_axes - 1))
}).create_node()
dft_node = IDFT(graph, dict(name=node.name,
in_ports_count=2)).create_node(
[node.in_node(0), axes])
# Slice a real part
begin_id = Const(graph, {
'value': int64_array([0, 0])
}).create_node()
end_id = Const(graph, {'value': int64_array([0, 1])}).create_node()
real = StridedSlice(
graph,
dict(name=node.name + '/real',
begin_mask=[0, 0],
end_mask=[0, 1],
shrink_axis_mask=[0, 0],
new_axis_mask=[0],
ellipsis_mask=[1, 0])).create_node(
[dft_node, begin_id, end_id])
squeeze_axis = Const(graph, {'value': -1}).create_node()
res = Squeeze(graph,
dict(name=node.name + '/squeeze')).create_node(
[real, squeeze_axis])
return [res.id]
else:
zero = Const(graph, {'value': 0.0}).create_node()
imag = Mul(graph, dict(name=node.name + '/imag')).create_node(
[node.in_node(0), zero])
cmplx = PackOp(graph,
dict(name=node.name + '/complex',
axis=-1)).create_node([node.in_node(0), imag])
axes = Const(graph, {
'value': int64_array(range(2, node.module.num_axes))
}).create_node()
dft_node = DFT(graph,
dict(name=node.name,
in_ports_count=2)).create_node([cmplx, axes])
return [dft_node.id]
def replace_op(self, graph: Graph, node: Node):
axis = Const(graph, {'value': int64_array([-1])}).create_node()
mvn = MVN(
graph,
dict(name=node.name + '/mvn',
eps=node.module.eps,
normalize_variance=True,
eps_mode='inside_sqrt')).create_node([node.in_node(0), axis])
weight = node.module.weight.detach().numpy()
bias = node.module.bias.detach().numpy()
w = Const(graph, {'value': weight}).create_node()
b = Const(graph, {'value': bias}).create_node()
mul = Mul(graph, dict(name=node.name + '/mul')).create_node([mvn, w])
add = Add(graph, dict(name=node.name + '/add')).create_node([mul, b])
return [add.id]
def replace_pattern(graph: Graph, match: dict):
node = match['conv']
# create Reshape before convolution
# shape = [in_shape[0], in_shape[1]/patch_stride, 1, patch_stride]
shape = Shape(graph, {}).create_node()
shape.in_port(0).connect(node.in_port(0).get_source())
split = create_op_with_const_inputs(graph, VariadicSplit, {
1: int64_array(0),
2: int64_array([1, -1])
}, {'out_ports_count': 2}, shape)
conv_patch_stride = Const(graph, {
'value': int64_array([node.patch_stride])
}).create_node()
pow_node = create_op_node_with_second_input(graph, Pow,
int64_array([-1]))
pow_node.in_port(0).connect(conv_patch_stride.out_port(0))
mul = Mul(graph, {}).create_node()
mul.in_port(0).connect(split.out_port(1))
mul.in_port(1).connect(pow_node.out_port(0))
const_1 = Const(graph, {'value': int64_array([1])}).create_node()
concat = Concat(graph, {'in_ports_count': 4, 'axis': 0}).create_node()
concat.in_port(0).connect(split.out_port(0))
concat.in_port(1).connect(mul.out_port(0))
concat.in_port(2).connect(const_1.out_port(0))
concat.in_port(3).connect(conv_patch_stride.out_port(0))
reshape_in = Reshape(graph, {
'name': '/Reshape/' + node.name
}).create_node()
reshape_in.in_port(1).connect(concat.out_port(0))
# create Reshape after Convolution
reshape_out = create_op_node_with_second_input(
graph, Reshape, int64_array([0, -1]),
{'name': node.name + '/Reshape/'})
# connect input_reshape_node
source = node.in_port(0).get_source()
node.in_port(0).get_connection().set_source(reshape_in.out_port(0))
reshape_in.in_port(0).connect(source)
# connect output_reshape_node
node.out_port(0).get_connection().set_source(reshape_out.out_port(0))
node.out_port(0).connect(reshape_out.in_port(0))
def replace_interpolate_pattern(graph: Graph, match: dict):
split = match['split']
scale = np.array([get_split_scale(split)], dtype=np.float32)
axis = int(split.in_port(1).get_connection().get_source().node.value)
split_node_name = split.name
axis_node = Const(graph, {'name': split_node_name + '/axis', 'value': int64_array([axis])}).create_node()
shape_node = Shape(graph, dict(name=split_node_name + '/Shape')).create_node()
scales_node = Const(graph, dict(name=split_node_name + '/scales', value=scale)).create_node()
mul_node = Mul(graph, dict(name=split_node_name + '/Mul')).create_node()
scales_node.out_port(0).connect(mul_node.in_port(1))
strided_slice_node = create_op_with_const_inputs(graph,
StridedSlice,
{1: int64_array([axis]), 2: int64_array([axis + 1])},
{
'name': split_node_name + '/StridedSlice',
'begin_mask': int64_array([1]),
'end_mask': int64_array([1]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])
})
shape_node.out_port(0).connect(strided_slice_node.in_port(0))
cast_shape_to_float = Cast(graph, {'dst_type': np.float32}).create_node()
strided_slice_node.out_port(0).connect(cast_shape_to_float.in_port(0))
cast_shape_to_float.out_port(0).connect(mul_node.in_port(0))
interp_node = Interpolate(graph,
dict(name=split_node_name + '/Interpolate',
mode='nearest',
antialias=0, pads_begin=int64_array([0]), pads_end=int64_array([0]),
coordinate_transformation_mode='half_pixel', nearest_mode='round_prefer_floor',
cube_coeff=-0.75, version='opset4', shape_calculation_mode='scales',
in_ports_count=4, maybe_part_of_sequence=True)).create_node()
floor_node = Floor(graph, {'name': split_node_name + '/Floor'}).create_node()
cast_mul_result_to_int = Cast(graph, {'dst_type': np.int64}).create_node()
mul_node.out_port(0).connect(floor_node.in_port(0))
floor_node.out_port(0).connect(cast_mul_result_to_int.in_port(0))
cast_mul_result_to_int.out_port(0).connect(interp_node.in_port(1))
scales_node.out_port(0).connect(interp_node.in_port(2))
axis_node.out_port(0).connect(interp_node.in_port(3))
match['concat'].out_port(0).get_connection().set_source(interp_node.out_port(0))
split_connection = split.in_port(0).get_connection()
split_connection.set_destination(interp_node.in_port(0))
split_connection.get_source().connect(shape_node.in_port(0))
def find_and_replace_pattern(self, graph: Graph):
for node in graph.get_op_nodes(op='ThresholdedRelu'):
name = node.soft_get('name', node.id)
greater = create_op_with_const_inputs(graph, Greater, {1: float_array([node.alpha])})
greater.in_port(0).connect(node.in_port(0).get_source())
float_greater = Cast(graph,
{'dst_type': data_type_str_to_np(graph.graph['cmd_params'].data_type)}).create_node()
greater.out_port(0).connect(float_greater.in_port(0))
mul = Mul(graph, {}).create_node()
node.out_port(0).get_connection().set_source(mul.out_port(0))
mul.in_port(0).connect(node.in_port(0).get_source())
mul.in_port(1).connect(float_greater.out_port(0))
rename_nodes([(node, name + '/TBR'), (mul, name)])
def replace_op(self, graph: Graph, node: Node):
mean = node.module.running_mean.detach().numpy()
var = node.module.running_var.detach().numpy()
weight = node.module.weight.detach().numpy()
bias = node.module.bias.detach().numpy()
w = weight / np.sqrt(var + node.module.eps)
b = bias - w * mean
shape = np.ones(node.module.dims, dtype=np.int32)
shape[1] = -1 # channels
w = Const(graph, {'value': w.reshape(shape)}).create_node()
b = Const(graph, {'value': b.reshape(shape)}).create_node()
mul = Mul(graph, dict(name=node.name + '/mul')).create_node(
[node.in_node(0), w])
add = Add(graph, dict(name=node.name + '/add')).create_node([mul, b])
return [add.id]
def replace_pattern(self, graph: Graph, match: dict):
quantize = match['quantize']
sum_node = Add(graph, dict()).create_node()
const = Const(graph, {'value': np.array(0.5)}).create_node()
mul_node = Mul(graph, dict()).create_node()
mul_node.in_port(0).connect(sum_node.out_port(0))
mul_node.in_port(1).connect(const.out_port(0))
quantize.in_port(1).get_connection().get_source().connect(
sum_node.in_port(0))
quantize.in_port(2).get_connection().get_source().connect(
sum_node.in_port(1))
quantize.in_port(1).disconnect()
quantize.in_port(2).disconnect()
mul_node.out_port(0).connect(quantize.in_port(1))
mul_node.out_port(0).connect(quantize.in_port(2))
def replace_op(self, graph: Graph, node: Node):
node_name = node.soft_get('name', node.id)
rename_node(node, node_name + '/TBR')
sqr_node = Mul(graph, {}).create_node()
reduce_sum_node = ReduceSum(
graph, {
'keep_dims': node.soft_get('keep_dims', 0),
'axis': node.soft_get('axis', None)
}).create_node()
sqrt_node = create_op_with_const_inputs(graph, Pow,
{1: float_array(0.5)})
rename_node(sqrt_node, node_name)
# Connect nodes
node.in_port(0).get_connection().set_destination(sqr_node.in_port(0))
sqr_node.in_port(0).get_connection().add_destination(
sqr_node.in_port(1))
sqr_node.out_port(0).connect(reduce_sum_node.in_port(0))
reduce_sum_node.out_port(0).connect(sqrt_node.in_port(0))
return [sqrt_node.id]
def replace_op(self, graph: Graph, node: Node):
# Add new nodes
const = Const(graph,
dict(value=np.array(-1, dtype=np.int32))).create_node()
negate = Mul(graph, {'name': node.name + '/negate_'}).create_node()
add = Add(graph, {'name': node.name + '/add_'}).create_node()
# Connect nodes
node.in_port(1).get_connection().set_destination(negate.in_port(0))
const.out_port(0).connect(negate.in_port(1))
node.in_port(0).get_connection().set_destination(add.in_port(1))
negate.out_port(0).connect(add.in_port(0))
# The "explicit" version of the return value is: [(out_node.id, 0)])
return [add.id]
def replace_interpolate_pattern(graph: Graph, match: dict):
split = match['split']
scale = int64_array([get_split_scale(split)])
axis = int(split.in_port(1).get_connection().get_source().node.value)
split_node_name = split.name
shape_node = Shape(graph,
dict(name=split_node_name + '/Shape_')).create_node()
scales_node = Const(graph,
dict(name=split_node_name + '/scales_',
value=scale)).create_node()
mul_node = Mul(graph, dict(name=split_node_name + '/Mul_')).create_node()
scales_node.out_port(0).connect(mul_node.in_port(1))
slice_begin = Const(
graph,
dict(name=split_node_name + '/slice_begin_',
value=int64_array([axis]))).create_node()
slice_end = Const(
graph,
dict(name=split_node_name + '/slice_end_',
value=int64_array([axis + 1]))).create_node()
strided_slice_node = StridedSlice(
graph, {
'name': split_node_name + '/StridedSlice_',
'begin_mask': int64_array([1]),
'end_mask': int64_array([1]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0]),
}).create_node([shape_node, slice_begin, slice_end])
strided_slice_node.out_port(0).connect(mul_node.in_port(0))
interp_node = Interpolate(
graph,
dict(name=split_node_name + '/Interpolate_',
axes=int64_array([axis]),
mode='nearest')).create_node()
mul_node.out_port(0).connect(interp_node.in_port(1))
match['concat'].out_port(0).get_connection().set_source(
interp_node.out_port(0))
split_connection = split.in_port(0).get_connection()
split_connection.set_destination(interp_node.in_port(0))
split_connection.get_source().connect(shape_node.in_port(0))
def replace_pattern(graph: Graph, match: [str, Node]):
node = match['sub']
# Add new nodes
negate_const = Const(
graph, dict(name=node.name + '/negate_const',
value=np.array(-1))).create_node()
negate = Mul(graph, {'name': node.name + '/negate_'}).create_node()
add = Add(graph, {'name': node.name + '/add_'}).create_node()
# Connect nodes
node.in_port(1).get_connection().set_destination(negate.in_port(0))
negate_const.out_port(0).connect(negate.in_port(1))
node.in_port(0).get_connection().set_destination(add.in_port(1))
negate.out_port(0).connect(add.in_port(0))
node.out_port(0).get_connection().set_source(add.out_port(0))
def replace_sub_graph(self, graph: Graph, match: dict):
resize_node = match['resize']
if match['mul_1'].in_node(1).value != match['mul_2'].in_node(1).value:
log.info('Pattern matched around resize op {} has different scale values.'.format(resize_node.name))
return
interpolate_node = Interpolate(graph, {'name': resize_node.name + '/Interpolate',
'mode': resize_node.mode, 'axes': int64_array([2, 3])}).create_node()
scale = match['mul_1'].in_node(1).value
scale_value = int64_array([scale, scale])
scale_const = Const(graph, {'value': scale_value, 'name': resize_node.name + '/Scale'}).create_node()
interpolated_shape = Mul(graph, {'name': resize_node.name + '/OutputShape'}).create_node()
match['slice'].out_port(0).connect(interpolated_shape.in_port(0))
scale_const.out_port(0).connect(interpolated_shape.in_port(1))
resize_node.in_port(0).get_connection().set_destination(interpolate_node.in_port(0))
interpolated_shape.out_port(0).connect(interpolate_node.in_port(1))
resize_node.out_port(0).get_connection().set_source(interpolate_node.out_port(0))
def replace_op(self, graph: Graph, node: Node):
# Create nodes
const_neg = Const(
graph, dict(value=np.array(-1),
name=node.name + '/negate_const_')).create_node()
negate = Mul(graph, {'name': node.name + '/negate_'}).create_node()
add = Add(graph, {'name': node.name + '/add_'}).create_node()
const = Const(graph, {'value': np.array(2)}).create_node()
squared = Pow(graph, {'name': node.name + '/squared_'}).create_node()
# Connect nodes
node.in_port(0).get_connection().set_destination(add.in_port(0))
node.in_port(1).get_connection().set_destination(negate.in_port(0))
const_neg.out_port(0).connect(negate.in_port(1))
negate.out_port(0).connect(add.in_port(1))
add.out_port(0).connect(squared.in_port(0))
const.out_port(0).connect(squared.in_port(1))
# The "explicit" version of the return value is: [(out_node.id, 0)])
return [squared.id]
def replace_op(self, graph: Graph, node: Node):
name = node.soft_get('name', node.id)
# create range of axes for MVN based on `start_axis` and rank of input
rank = Rank(graph, {'name': name + '/Rank'}).create_node()
rng = create_op_with_const_inputs(graph, Range, {
0: int64_array(2),
2: int64_array(1)
}, {
'name': name + '/Range',
'output_type': np.int64
})
mvn = MVN(
graph, {
'eps': node.epsilon,
'eps_mode': 'inside_sqrt',
'normalize_variance': 1,
'name': name + '/Ins_Norm/MVN_',
}).create_node()
node.in_port(0).get_connection().set_destination(mvn.in_port(0))
rng.out_port(0).connect(mvn.in_port(1))
mul = Mul(graph, {
'axis': 1,
'name': name + '/Ins_Norm/mul_'
}).create_node()
mvn.out_port(0).connect(mul.in_port(0))
node.in_port(1).get_connection().set_destination(mul.in_port(1))
add = Add(graph, {
'axis': 1,
'name': name + '/Ins_Norm/add_'
}).create_node()
mul.out_port(0).connect(add.in_port(0))
node.in_port(2).get_connection().set_destination(add.in_port(1))
mvn.in_port(0).get_connection().add_destination(rank.in_port(0))
rng.in_port(1).connect(rank.out_port(0))
rename_nodes([(node, name + '/TBD'), (add, name)])
return [add.id]
def replace_op(self, graph: Graph, node: Node):
# split input to (i_part, f_part, c_part, o_part, ct_1)
split_node_axis = Const(graph, {'value': np.int64(1)}).create_node()
split_node = Split(graph, {
'name': 'Split_lstm_input_',
'num_splits': 5
}).create_node()
node.in_port(0).get_connection().set_destination(split_node.in_port(0))
split_node.in_port(1).connect(split_node_axis.out_port(0))
# i_t = Sigmoid(i_part + w_ic*ct_1)
i_scale_attrs = {'name': 'i_scaleshift', 'bias_term': False}
i_scale = ScaleShiftOp(graph, i_scale_attrs).create_node()
input_as_const(i_scale, i_scale_attrs, 1, 'weights', node.i_weights)
split_node.out_port(4).connect(i_scale.in_port(0))
sum_i_c = Add(graph, {'name': 'sum_i_c_'}).create_node()
split_node.out_port(0).connect(sum_i_c.in_port(0))
i_scale.out_port(0).connect(sum_i_c.in_port(1))
i_sigmoid = Sigmoid(graph, {'name': 'i_sigmoid'}).create_node()
sum_i_c.out_port(0).connect(i_sigmoid.in_port(0))
# f_t = Sigmoid(f_part + w_fc*ct_1)
f_scale_attrs = {'name': 'f_scaleshift', 'bias_term': False}
f_scale = ScaleShiftOp(graph, f_scale_attrs).create_node()
input_as_const(f_scale, f_scale_attrs, 1, 'weights', node.f_weights)
split_node.out_port(4).connect(f_scale.in_port(0))
sum_f_c = Add(graph, {'name': 'sum_f_c_'}).create_node()
split_node.out_port(1).connect(sum_f_c.in_port(0))
f_scale.out_port(0).connect(sum_f_c.in_port(1))
f_sigmoid = Sigmoid(graph, {'name': 'f_sigmoid'}).create_node()
sum_f_c.out_port(0).connect(f_sigmoid.in_port(0))
# c_t = f_t*ct_1 + i_t * tanh(c_part)
c_tanh = Tanh(graph, {'name': 'c_tanh'}).create_node()
split_node.out_port(2).connect(c_tanh.in_port(0))
prod_i_c_tanh = Mul(graph, {'name': 'prod_i_c_tanh_'}).create_node()
i_sigmoid.out_port(0).connect(prod_i_c_tanh.in_port(0))
c_tanh.out_port(0).connect(prod_i_c_tanh.in_port(1))
prod_f_ct_1 = Mul(graph, {'name': 'prod_f_ct_1_'}).create_node()
f_sigmoid.out_port(0).connect(prod_f_ct_1.in_port(0))
split_node.out_port(4).connect(prod_f_ct_1.in_port(1))
sum_f_i = Add(graph, {'name': 'sum_f_i_'}).create_node()
prod_f_ct_1.out_port(0).connect(sum_f_i.in_port(0))
prod_i_c_tanh.out_port(0).connect(sum_f_i.in_port(1))
# o_t = Sigmoid(o_part + w_oc*c_t)
o_scale_attrs = {'name': 'o_scaleshift', 'bias_term': False}
o_scale = ScaleShiftOp(graph, o_scale_attrs).create_node()
input_as_const(o_scale, o_scale_attrs, 1, 'weights', node.o_weights)
sum_f_i.out_port(0).connect(o_scale.in_port(0))
sum_o_c = Add(graph, {'name': 'sum_o_c_'}).create_node()
split_node.out_port(3).connect(sum_o_c.in_port(0))
o_scale.out_port(0).connect(sum_o_c.in_port(1))
o_sigmoid = Sigmoid(graph, {'name': 'o_sigmoid'}).create_node()
sum_o_c.out_port(0).connect(o_sigmoid.in_port(0))
# m_t = o_t * Tanh(c_t)
c_t_tanh = Tanh(graph, {'name': 'c_t_tanh'}).create_node()
sum_f_i.out_port(0).connect(c_t_tanh.in_port(0))
prod_o_c_t_tanh = Mul(graph, {
'name': 'prod_o_c_t_tanh_'
}).create_node()
o_sigmoid.out_port(0).connect(prod_o_c_t_tanh.in_port(0))
c_t_tanh.out_port(0).connect(prod_o_c_t_tanh.in_port(1))
# add concat to create 1 output
concat = Concat(graph, {'name': 'Concat_c_m'}).create_node()
concat.add_sequence_of_ports('in', range(2))
sum_f_i.out_port(0).connect(concat.in_port(0))
prod_o_c_t_tanh.out_port(0).connect(concat.in_port(1))
return [concat.id]
def replace_pattern(self, graph: Graph, match: Dict[str, Node]):
log.debug('UpsampleToResample is triggered')
upsample = match['upsample']
input_shape = upsample.in_port(0).data.get_shape()
input_shape_rank = len(input_shape)
if input_shape_rank not in [4, 5]:
log.warning('The input shape is not 4D or 5D for op {}'.format(
upsample.soft_get('name')))
return
if len(upsample.in_nodes()) == 2:
if upsample.in_node(1).value is None:
return
scales = upsample.in_node(1).value
assert scales.shape == (4, )
if not (math.isclose(scales[0], 1, rel_tol=1e-5)
and math.isclose(scales[1], 1, rel_tol=1e-5)):
return
height_scale = scales[2]
width_scale = scales[3]
else:
height_scale = upsample['height_scale']
width_scale = upsample['width_scale']
if 1 in upsample.in_ports() and not upsample.in_port(1).disconnected():
upsample.in_port(1).disconnect()
factor = Const(graph, {
'value': np.array([height_scale, width_scale])
}).create_node()
shape = Shape(graph, {'name': upsample.name + '/0_port'}).create_node()
layout = graph.graph['layout']
if input_shape_rank == 4:
begin = Const(graph, {
'value':
int64_array([get_height_dim(layout, input_shape_rank)])
}).create_node()
else:
begin = Const(graph, {
'value':
int64_array([get_depth_dim(layout, input_shape_rank)])
}).create_node()
end = Const(graph, {
'value':
int64_array([get_width_dim(layout, input_shape_rank) + 1])
}).create_node()
stride = Const(graph, {'value': int64_array([1])}).create_node()
ss = StridedSlice(
graph, {
'name': upsample.name + '/ss_0_port',
'begin_mask': np.array([1]),
'end_mask': np.array([0]),
'new_axis_mask': np.array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])
}).create_node()
mul = Mul(graph, {
'name': upsample.name + '/factor_mul_'
}).create_node()
source = upsample.in_port(0).get_connection().get_source()
source.connect(shape.in_port(0))
shape.out_port(0).connect(ss.in_port(0))
begin.out_port(0).connect(ss.in_port(1))
end.out_port(0).connect(ss.in_port(2))
stride.out_port(0).connect(ss.in_port(3))
ss.out_port(0).connect(mul.in_port(0))
factor.out_port(0).connect(mul.in_port(1))
# Create Interpolate operation
if input_shape_rank == 4:
axes = int64_array([
get_height_dim(layout, input_shape_rank),
get_width_dim(layout, input_shape_rank)
])
else:
axes = int64_array([
get_depth_dim(layout, input_shape_rank),
get_height_dim(layout, input_shape_rank),
get_width_dim(layout, input_shape_rank)
])
resample_op = Interpolate(
graph,
dict(name='Interpolate/{}'.format(upsample.name),
axes=axes,
mode=upsample.attrs()['mode'],
antialias=0,
convert_to_resample=True)).create_node()
upsample.add_input_port(1, skip_if_exist=True)
assert upsample.in_port(1).disconnected()
mul.out_port(0).connect(resample_op.in_port(1))
upsample.in_port(0).get_connection().set_destination(
resample_op.in_port(0))
upsample.out_port(0).get_connection().set_source(
resample_op.out_port(0))
def replace_pattern(self, graph: Graph, match: dict):
merge = match['merge']
power = Pow(graph, {
'name': merge.name + '/reciprocal_',
'type': 'PNORM'
}).create_node()
const1 = Const(graph, {
'value': -1.0,
'name': merge.name + '/negate_const'
}).create_node()
merge.in_port(0).get_connection().set_destination(power.in_port(0))
const1.out_port(0).connect(power.in_port(1))
concat_node = Concat(
graph, {
'axis': 0,
'name': merge.name + '/Concat_',
'override_output_shape': True
}).create_node()
const3 = Const(graph, {
'name': merge.name + '/const_reduce',
'value': 0
}).create_node()
for ii, idx in enumerate(
range(merge.significant, merge.to_significant + 1, 1)):
const_node = Const(
graph, {
'value': float_array(math.pow(10.0, idx)),
'name': merge.name + '/Const_' + ii.__str__()
}).create_node()
mul_node = Mul(graph, {
'name': merge.name + '/Mul_' + ii.__str__()
}).create_node()
const_node.out_port(0).connect(mul_node.in_port(0))
power.out_port(0).connect(
mul_node.in_port(1)) # connect to the graph node
mul_node2 = Mul(graph, {
'name': merge.name + '/Mul_Div_' + ii.__str__()
}).create_node()
const_node2 = Const(
graph, {
'value': float_array(math.pow(10.0, -1 * idx)),
'name': merge.name + '/Const_Pow_' + ii.__str__()
}).create_node()
cast_node = Cast(
graph, {
'name': merge.name + '/Cast_' + idx.__str__(),
'dst_type': np.float32
}).create_node()
mul_node.out_port(0).connect(cast_node.in_port(0))
const_node2.out_port(0).connect(mul_node2.in_port(1))
cast_node.out_port(0).connect(mul_node2.in_port(0))
concat_node.add_input_port(ii, skip_if_exist=True)
concat_node.in_port(ii).get_connection().set_source(
mul_node2.out_port(0))
reducesum_node = ReduceMean(
graph, {
'name': merge.id + '/_pnorm_reduced_sum',
'keep_dims': False,
'in_ports_count': 2,
'need_shape_inference': None,
'infer': reduce_infer
}).create_node()
const3.out_port(0).connect(reducesum_node.in_port(1))
reducesum_node.in_port(0).get_connection().set_source(
concat_node.out_port(0))
reshape = Reshape(graph, {
'name': merge.name + '/Reshape_Node'
}).create_node()
reshape_dim = Const(graph, {
'value': np.array([1, 5]),
'name': merge.id + '/Reshape_Dim'
}).create_node()
reducesum_node.out_port(0).connect(reshape.in_port(0))
reshape.in_port(1).connect(reshape_dim.out_port(0))
merge.out_port(0).get_connection().set_source(reshape.out_port(0))
def replace_pattern(self, graph: Graph, match: dict):
assert match['operator'].has('multiplication_transparent_ports')
quantize = match['quantize']
port = match['operator'].input_ports_with(match['quantized'])
assert len(port) >= 1
if len(port) > 1:
log.debug(
'BinarizeWeightsM1P1 cannot apply transformation for data {} because it consumed more'
' than once'.format(match['quantized'].name))
return
assert len(port) == 1
port = port[0]
applicable = [
pair for pair in match['operator'].multiplication_transparent_ports
if pair[0] == port
]
if len(applicable) == 0:
return
# Look at 3-rd and 4-th inputs of FakeQuantize -- they have constants that should be passed through.
# Assume that the constant that should be passed through is a scalar.
output_low = quantize.in_node(3)
output_high = quantize.in_node(4)
assert len(output_low.out_nodes()) == 1
assert len(output_high.out_nodes()) == 1
if not output_low.has_valid('value') and not output_high.has_valid(
'value'):
return
output_low = output_low.value
output_high = output_high.value
operator = match['operator']
weights = operator.in_node(1).value
weights_rounded = np.round(weights)
weights_consistent = np.all(np.isclose(weights, weights_rounded)) and \
set(np.unique(weights_rounded)).issubset({-1, 1})
if weights_consistent and np.all(np.isclose(output_low, 0)) and np.all(
np.isclose(output_high, 1)):
reduction_indices = set(range(len(weights.shape))) - set(
[operator.output_feature_channel])
weights_reduced = np.add.reduce(weights,
axis=tuple(reduction_indices))
weights_reduced = weights_reduced.reshape(
[len(weights_reduced), 1, 1]) # FIXME: works for NCHW only
add_term = Const(graph, {'value': weights_reduced}).create_node()
add = Add(graph, {}).create_node()
add.in_port(1).connect(add_term.out_port(0))
mul_term = Const(graph, {'value': np.array(0.5)}).create_node()
mul = Mul(graph, {}).create_node()
mul.in_port(1).connect(mul_term.out_port(0))
add.out_port(0).connect(mul.in_port(0))
operator.out_port(0).get_connection().set_source(mul.out_port(0))
add.in_port(0).connect(operator.out_port(0))
operator['pad_value'] = float(-1.0)
elif weights_consistent and np.all(np.isclose(
output_low, -1)) and np.all(np.isclose(output_high, +1)):
pass
else:
log.debug(
'ConvToBinaryConv: cannot apply transformation because input range is neither in [0, +1] nor '
'in [-1, +1].')
return
operator['type'] = 'BinaryConvolution'
operator['mode'] = 'xnor-popcount'
operator['pad_value'] = operator.soft_get('pad_value', float(0))
operator['input'] = operator.in_node(0).shape[1]
# Weights are not bit-packed yet; there should be a separate transformation to do that
assert output_low.size == 1
assert output_high.size == 1
output_low = quantize.in_node(3)
output_high = quantize.in_node(4)
# Make sure that low/high values are exactly 0/1
output_low.value = np.zeros(output_low.shape)
output_high.value = np.ones(output_high.shape)
def parse_specifier(string, graph, layer_node_map):
pos = string.find(b'(')
if pos == -1:
# node name
input_name = str(string.split(b' ')[0]).strip('b').replace(
"\'", '').replace('\\n', '')
if input_name not in layer_node_map:
node_name = graph.unique_id(prefix=input_name)
graph.add_node(node_name, parameters=[], op="", kind='op')
layer_node_map[input_name] = node_name
else:
node_name = layer_node_map[input_name]
return node_name
spec = string[:pos]
args = get_args_for_specifier(string[pos:])
if spec == b'Append':
nodes = []
for i in range(len(args)):
nodes.append(parse_specifier(args[i], graph, layer_node_map))
layer_name = 'Append_'
for node in nodes:
layer_name = layer_name + node + "_"
if layer_name not in layer_node_map:
concat_name = graph.unique_id(prefix=layer_name)
graph.add_node(concat_name,
parameters=None,
op='concat',
kind='op')
layer_node_map[layer_name] = concat_name
i = 0
Node(graph,
concat_name).add_sequence_of_ports('in', range(len(nodes)))
for node in nodes:
out_port = len(Node(graph, node).out_nodes())
Node(graph, node).add_output_port(out_port)
graph.create_edge(
Node(graph, node), Node(graph, concat_name), out_port, i,
create_edge_attrs(node, concat_name, node, i, out_port))
i = i + 1
else:
concat_name = layer_node_map[layer_name]
return concat_name
elif spec == b'Offset':
node = parse_specifier(args[0], graph, layer_node_map)
t = int(args[1])
if len(args) > 2:
raise Error("ModelOptimizer supports only 2 arguments for Offset")
layer_name = 'Offset_' + node + '_'
if t < 0:
layer_name = layer_name + '_' + str(-t)
else:
layer_name = layer_name + str(t)
if layer_name not in layer_node_map:
memory_name = graph.unique_id(prefix=layer_name)
layer_node_map[layer_name] = memory_name
memory_name_2 = memory_name + '_out'
graph.add_node(memory_name,
parameters=dict(t=t,
pair_name=memory_name_2,
has_default=False),
op='MemoryOffset',
kind='op')
out_port = len(Node(graph, node).out_nodes())
in_port = len(Node(graph, memory_name).in_nodes())
Node(graph, memory_name).add_input_port(in_port)
Node(graph, node).add_output_port(out_port, skip_if_exist=True)
graph.create_edge(
Node(graph, node), Node(graph, memory_name), out_port, in_port,
create_edge_attrs(node, memory_name, node, in_port, out_port))
else:
memory_name = layer_node_map[layer_name]
return memory_name
elif spec == b'Sum':
nodes = []
for i in range(len(args)):
nodes.append(parse_specifier(args[i], graph, layer_node_map))
layer_name = 'Sum_'
for node in nodes:
layer_name = layer_name + node + "_"
if layer_name not in layer_node_map:
sum_name = graph.unique_id(prefix=layer_name)
graph.add_node(sum_name, parameters=None, op='Add', kind='op')
layer_node_map[layer_name] = sum_name
else:
sum_name = layer_node_map[layer_name]
for i, node in enumerate(nodes):
out_port = len(Node(graph, node).out_nodes())
Node(graph, node).add_output_port(out_port, skip_if_exist=True)
Node(graph, sum_name).add_input_port(i)
graph.add_edge(node, sum_name,
**create_edge_attrs(node, sum_name, node, i))
return sum_name
elif spec == b'IfDefined':
node_id = parse_specifier(args[0], graph, layer_node_map)
node = Node(graph, node_id)
if node.op == 'MemoryOffset':
node['parameters']['has_default'] = True
return node_id
elif spec == b'ReplaceIndex':
node = parse_specifier(args[0], graph, layer_node_map)
return node
elif spec == b'Scale':
node_name = parse_specifier(args[1], graph, layer_node_map)
scale_value = float(args[0])
layer_name = '{}/Mul/{}'.format(node_name, scale_value)
if layer_name not in layer_node_map:
scale_name = graph.unique_id(prefix=layer_name)
scale_node = Mul(graph, {'name': scale_name}).create_node()
layer_node_map[layer_name] = scale_name
scale_const_name = 'Const_{}'.format(scale_value)
const_node = Const(graph, {
'name': scale_const_name,
'value': float_array([scale_value])
}).create_node()
node = Node(graph, node_name)
graph.create_edge(
const_node, scale_node, 0, 0,
create_edge_attrs(const_node.id, scale_node.id, const_node.id))
out_port = len(node.out_nodes())
graph.create_edge(
node, scale_node, out_port, 1,
create_edge_attrs(node_name, scale_node.id, node_name, 1,
out_port))
else:
scale_name = layer_node_map[layer_name]
return scale_name
def _fused_batch_norm_decomposition(graph: Graph,
tinput: Port,
toutput: Port,
gamma: Port,
beta: Port,
mean: np.ndarray,
variance: np.ndarray,
can_be_fused=True):
"""
This is common function for TF, Caffe and MXNet
It creates Mul->Add->Mul->Add sub graph
"""
batch_norm_name = tinput.get_connection().get_destination().node.name
# Create first Mul & Add operations
mul1_node = Mul(
graph, dict(name=batch_norm_name + "/mean",
can_be_fused=can_be_fused)).create_node()
add1_node = Add(
graph,
dict(name=batch_norm_name + "/variance",
can_be_fused=can_be_fused)).create_node()
const_mul1_node = Const(graph, dict(name="data_mul_",
value=np.array(mean))).create_node()
const_add1_node = Const(graph,
dict(name="data_add_",
value=np.array(variance))).create_node()
# Broadcast const from scalar
# We can broadcast only when const.value is scalar
if gamma.data.get_shape()[0] != gamma.data.get_value().shape[0]:
value = gamma.data.get_value()
value.resize(gamma.data.get_shape()).fill(value[0])
gamma.data.set_value(value)
# Create second Mul & Add
mul2_node = Mul(
graph, dict(name=batch_norm_name + "/gamma",
can_be_fused=can_be_fused)).create_node()
add2_node = Add(
graph, dict(name=batch_norm_name + "/beta",
can_be_fused=can_be_fused)).create_node()
# Connect edges Mul1->Add1->Mul2->Add2
tinput.get_connection().set_destination(mul1_node.in_port(0))
mul1_node.in_port(1).get_connection().set_source(
const_mul1_node.out_port(0))
add1_node.in_port(0).get_connection().set_source(mul1_node.out_port(0))
add1_node.in_port(1).get_connection().set_source(
const_add1_node.out_port(0))
mul2_node.in_port(0).get_connection().set_source(add1_node.out_port(0))
gamma.get_connection().set_destination(mul2_node.in_port(1))
add2_node.in_port(0).get_connection().set_source(mul2_node.out_port(0))
beta.get_connection().set_destination(add2_node.in_port(1))
toutput.get_connection().set_source(add2_node.out_port(0))
def convert_scale_shift_to_mul_add(graph: Graph):
nodes = graph.get_op_nodes(op='ScaleShift')
for node in nodes:
if node.soft_get('can_be_fused') is False:
continue
ports_count = len(node.in_ports())
input_port = node.in_port(0)
scale_port = node.in_port(1) if ports_count > 1 and not node.in_port(
1).disconnected() else None
shift_port = node.in_port(2) if ports_count > 2 and not node.in_port(
2).disconnected() else None
output_port = node.out_port(0)
has_biases = True
has_weights = True
# We don't need zero biases
if shift_port is None or (
shift_port.data.get_value() is not None
and all([x == 0 for x in shift_port.data.get_value()])):
has_biases = False
# We don't need weights with ones
if scale_port is None or (
scale_port.data.get_value() is not None
and all([x == 1 for x in scale_port.data.get_value()])):
has_weights = False
mul_op = Mul(graph, dict(name=node.name + "/Mul_"))
add_op = Add(graph, dict(name=node.name + "/Add_"))
# Expand dims for current layout
broadcast_dims_cnt = len(input_port.data.get_shape(
)) - 2 if graph.graph['layout'] == 'NCHW' else 0
# In case if we have constant weights/biases we have to broadcast them according to graph layout
# otherwise we insert Reshape with broadcast dim attribute.
def broadcast_value(port):
value = np.array(port.data.get_value())
for idx in range(broadcast_dims_cnt):
value = np.expand_dims(value, axis=-1)
port.data.set_value(value)
def broadcast_with_reshape(port):
input_shape = input_port.data.get_shape()
reshape_dims = np.zeros(len(input_shape), dtype=np.int64)
for i in range(0, node.axis):
reshape_dims[i] = 1
data_shape = port.data.get_shape()
for i in range(node.axis, node.axis + len(data_shape)):
reshape_dims[i] = data_shape[i - node.axis]
for i in range(node.axis + len(data_shape), len(input_shape)):
reshape_dims[i] = 1
reshape = create_op_node_with_second_input(
graph, Reshape, reshape_dims,
dict(name=port.node.name + "/Broadcast_"))
port.get_connection().set_destination(reshape.in_port(0))
reshape.out_port(0).connect(port)
if has_weights and scale_port.data.get_value() is not None:
broadcast_value(scale_port)
elif has_weights:
broadcast_with_reshape(scale_port)
if has_biases and shift_port.data.get_value() is not None:
broadcast_value(shift_port)
elif has_biases:
broadcast_with_reshape(shift_port)
if has_biases and has_weights:
# Connect input->mul->out->add->out
add_node = add_op.create_node()
mul_node = mul_op.create_node()
# Connect Mul operation with inputs
input_port.get_connection().set_destination(mul_node.in_port(0))
scale_port.get_connection().set_destination(mul_node.in_port(1))
# Connect Add operation with inputs
mul_node.out_port(0).connect(add_node.in_port(0))
shift_port.get_connection().set_destination(add_node.in_port(1))
output_port.get_connection().set_source(add_node.out_port(0))
elif has_weights:
# Connect input->mul->out
mul_node = mul_op.create_node()
# Connect Mul operation with inputs
input_port.get_connection().set_destination(mul_node.in_port(0))
scale_port.get_connection().set_destination(mul_node.in_port(1))
output_port.get_connection().set_source(mul_node.out_port(0))
elif has_biases:
# Connect input->add->out
add_node = add_op.create_node()
# Connect Add operation with inputs
input_port.get_connection().set_destination(add_node.in_port(0))
shift_port.get_connection().set_destination(add_node.in_port(1))
output_port.get_connection().set_source(add_node.out_port(0))
else:
# Connect input->out
producer_port = input_port.get_source()
input_port.disconnect()
output_port.get_connection().set_source(producer_port)
def replace_pattern(graph: Graph, match: dict):
log.debug(
'================== GNMTBeforeConditionFind ==================')
input_sequence_lengths = match['Max'].in_port(0).get_source().node
encoder_sequence_lengths = looking_for_op_in_list([
port.node
for port in input_sequence_lengths.out_port(0).get_destinations()
], 'Identity')
# Looking for Sequence_length node in encoder looks like:
# Sequence_length -> CheckSeqLen -> Max -> Maximum -> Minimum
check_seq_len = looking_for_op_in_list([
port.node for port in encoder_sequence_lengths.out_port(
0).get_destinations()
], 'Identity')
max = looking_for_op_in_list([
port.node for port in check_seq_len.out_port(0).get_destinations()
], 'ReduceMax')
maximum = max.out_port(0).get_destinations()[0].node
assert maximum.op == 'Maximum'
minimum = maximum.out_port(0).get_destinations()[0].node
assert minimum.op == 'Minimum'
tensor_seq_len = looking_for_op_in_list([
minimum.in_port(port).get_source().node
for port in minimum.in_ports()
], 'StridedSlice')
# Create node for multiplying seq_len by 2
const = Const(graph, {
'name': 'FakeSeqLenMultiplyer',
'value': np.array(2)
}).create_node()
mul_op = Mul(graph, {'name': 'FakeSeqLen'}).create_node()
const.out_port(0).get_connection().set_destination(mul_op.in_port(1))
tensor_seq_len.out_port(0).get_connection().add_destination(
mul_op.in_port(0))
# Connect seq_len * 2 to TensorArray from GNMT loop
ta_writes = [
port.node
for port in match['Identity_1'].out_port(0).get_destinations()
if port.node.op == 'TensorArrayWriteV3'
]
for ta_write in ta_writes:
ta = ta_write.in_port(0).get_source().node.in_port(
0).get_source().node
ta.in_port(0).disconnect()
ta.in_port(0).get_connection().set_source(mul_op.out_port(0))
if not graph.graph['cmd_params'].static_shape:
log.error(
"Model can not be translated in a reshape-able way.\n"
"Model Optimizer key static_shape was turned on to prevent related errors.\n"
"There will be no success changing input shapes of the model with the help of "
"InferenceEngine reshape method",
extra={'is_warning': True})
graph.graph['cmd_params'].static_shape = True
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch):
reshape_classes_node = create_op_node_with_second_input(
graph, Reshape, int64_array([0, -1]),
dict(name='do_reshape_classes'),
match.single_input_node(1)[0])
priors_node = match.single_input_node(2)[0]
placeholder = [
Node(graph, node_id) for node_id in graph.nodes()
if Node(graph, node_id).op == 'Parameter'
][0]
im_height = placeholder.shape[1]
im_width = placeholder.shape[2]
# scale prior boxes to the [0, 1] interval
priors_scale_const_node = Const(
graph, {
'value':
np.array(
[1 / im_width, 1 / im_height, 1 / im_width, 1 / im_height])
}).create_node([])
priors_scale_node = Mul(graph, {
'name': 'scale_priors'
}).create_node([priors_node, priors_scale_const_node])
# calculate prior boxes widths and heights
split_node = SplitV(graph, {
'axis': 2,
'size_splits': [1, 1, 1, 1],
'out_ports_count': 4
}).create_node([priors_scale_node])
priors_width_node = Sub(
graph, dict(name=split_node.name + '/sub_2-0_')).create_node([
(split_node, 2), (split_node, 0)
])
priors_height_node = Sub(graph, dict(name=split_node.name +
'/sub_3-1_')).create_node([
(split_node, 3),
(split_node, 1)
])
# concat weights and heights into a single tensor and multiple with the box coordinates regression values
concat_width_height_node = Concat(graph, {
'name': 'concat_priors_width_height',
'axis': -1,
'in_ports_count': 4
}).create_node([
priors_width_node, priors_height_node, priors_width_node,
priors_height_node
])
applied_width_height_regressions_node = Mul(graph, {
'name': 'final_regressions'
}).create_node(
[concat_width_height_node,
match.single_input_node(0)[0]])
# reshape to 2D tensor as Inference Engine Detection Output layer expects
reshape_regression_node = create_op_node_with_second_input(
graph, Reshape, int64_array([0, -1]),
dict(name='reshape_regression'),
applied_width_height_regressions_node)
detection_output_op = DetectionOutput(
graph, match.custom_replacement_desc.custom_attributes)
detection_output_op.attrs['old_infer'] = detection_output_op.attrs[
'infer']
detection_output_op.attrs['infer'] = __class__.do_infer
detection_output_node = detection_output_op.create_node(
[reshape_regression_node, reshape_classes_node, priors_scale_node],
dict(name=detection_output_op.attrs['type'],
clip=1,
normalized=1,
variance_encoded_in_target=0))
return {'detection_output_node': detection_output_node}
def replace_resize(graph: Graph, resize: Node):
log.debug("Converting of ONNX Resize-11 to Interpolate-4 "
"is triggered for node {}.".format(
resize.soft_get('name', resize.id)))
input_shape = resize.in_port(0).data.get_shape()
input_rank = len(input_shape)
resize_name = resize.soft_get('name', resize.id)
if input_rank not in {4, 5}:
log.warning(
'The input shape is not 4D or 5D for op with name {}'.format(
resize_name))
return
num_of_inputs = len([
port for port in resize.in_ports().values() if not port.disconnected()
])
assert num_of_inputs in {3, 4}, \
"Number of inputs of ONNXResize (with name {}) should be equal to 3 or 4".format(resize_name)
assert resize.soft_get('coordinate_transformation_mode') != 'tf_crop_and_resize', \
'Mode tf_crop_and_resize is not supported for op {} with name {}'.format(resize.op, resize_name)
layout = graph.graph['layout']
if input_rank == 4:
begin_dim = get_height_dim(layout, input_rank)
end_dim = get_width_dim(layout, input_rank) + 1
else:
begin_dim = get_depth_dim(layout, input_rank)
end_dim = get_width_dim(layout, input_rank) + 1
sizes_ss = create_op_with_const_inputs(
graph, StridedSlice, {
1: int64_array([begin_dim]),
2: int64_array([end_dim]),
3: int64_array([1])
}, {
'name': resize_name + '/StridedSlice_sizes',
'begin_mask': int64_array([1]),
'end_mask': int64_array([1]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])
})
scales_ss = create_op_with_const_inputs(
graph, StridedSlice, {
1: int64_array([begin_dim]),
2: int64_array([end_dim]),
3: int64_array([1])
}, {
'name': resize_name + '/StridedSlice_scales',
'begin_mask': int64_array([1]),
'end_mask': int64_array([1]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])
})
axes_node = Const(
graph, {
'name': resize_name + '/axis',
'value': int64_array(np.arange(begin_dim, end_dim))
}).create_node()
shape_calculation_mode = 'scales' if num_of_inputs == 3 else 'sizes'
interpolate_node = Interpolate(
graph, {
'version': 'opset4',
'mode': convert_mode(resize.mode),
'coordinate_transformation_mode':
resize.coordinate_transformation_mode,
'cube_coeff': resize.cube_coeff,
'nearest_mode': resize.nearest_mode,
'pads_begin': int64_array([0]),
'pads_end': int64_array([0]),
'antialias': 0,
'shape_calculation_mode': shape_calculation_mode,
'in_ports_count': 4
}).create_node()
axes_node.out_port(0).connect(interpolate_node.in_port(3))
shape_of = Shape(graph, {'name': resize_name + '/ShapeOf'}).create_node()
add_node = create_op_with_const_inputs(graph, Add,
{1: float_array([1.0e-5])},
{'name': resize_name + '/Add'})
input_data_type = data_type_str_to_np(graph.graph['cmd_params'].data_type)
if num_of_inputs == 3:
cast_shape_to_float = Cast(graph, {
'dst_type': input_data_type
}).create_node()
mul_node = Mul(graph, {'name': resize_name + '/Mul'}).create_node()
shape_of.out_port(0).connect(cast_shape_to_float.in_port(0))
cast_shape_to_float.out_port(0).connect(mul_node.in_port(0))
cast_add_result_to_int = Cast(graph, {
'dst_type': np.int64
}).create_node()
floor_node = Floor(graph, {
'name': resize_name + '/Floor'
}).create_node()
mul_node.out_port(0).connect(add_node.in_port(0))
add_node.out_port(0).connect(floor_node.in_port(0))
floor_node.out_port(0).connect(cast_add_result_to_int.in_port(0))
cast_add_result_to_int.out_port(0).connect(sizes_ss.in_port(0))
sizes_ss.out_port(0).connect(interpolate_node.in_port(1))
scales_ss.out_port(0).connect(interpolate_node.in_port(2))
connection_of_resize_input = resize.in_port(0).get_connection()
connection_of_resize_input.set_destination(interpolate_node.in_port(0))
connection_of_scales = resize.in_port(2).get_connection()
connection_of_scales.set_destination(scales_ss.in_port(0))
connection_of_resize_input.get_source().connect(shape_of.in_port(0))
connection_of_scales.get_source().connect(mul_node.in_port(1))
else:
cast_shape_to_float = Cast(graph, {
'dst_type': input_data_type
}).create_node()
cast_sizes_to_float = Cast(graph, {
'dst_type': input_data_type
}).create_node()
div_node = Div(graph, {'name': resize_name + '/Div'}).create_node()
cast_sizes_to_float.out_port(0).connect(div_node.in_port(0))
cast_shape_to_float.out_port(0).connect(div_node.in_port(1))
shape_of.out_port(0).connect(cast_shape_to_float.in_port(0))
div_node.out_port(0).connect(add_node.in_port(0))
add_node.out_port(0).connect(scales_ss.in_port(0))
scales_ss.out_port(0).connect(interpolate_node.in_port(2))
sizes_ss.out_port(0).connect(interpolate_node.in_port(1))
connection_of_resize_input = resize.in_port(0).get_connection()
connection_of_resize_input.set_destination(interpolate_node.in_port(0))
connection_of_sizes = resize.in_port(3).get_connection()
connection_of_sizes.set_destination(sizes_ss.in_port(0))
connection_of_resize_input.get_source().connect(shape_of.in_port(0))
connection_of_sizes.get_source().connect(
cast_sizes_to_float.in_port(0))
rename_nodes([(resize, resize_name + '/delete'),
(interpolate_node, resize_name)])
resize.out_port(0).get_connection().set_source(
interpolate_node.out_port(0))
def replace_pattern(graph: Graph, match: dict):
node = match['conv']
node_name = node.soft_get('name', node.id)
# create Reshape before convolution
# shape = [in_shape[0], in_shape[1]/patch_stride, 1, patch_stride]
shape = Shape(graph, {'name': node_name + '/Shape'}).create_node()
shape.in_port(0).connect(node.in_port(0).get_source())
split = create_op_with_const_inputs(graph, VariadicSplit, {
1: int64_array(0),
2: int64_array([1, -1])
}, {
'name': shape.name + '/split_batch',
'out_ports_count': 2
}, shape)
pow_node = create_op_node_with_second_input(
graph, Pow, int64_array([-1]),
{'name': node_name + '/patch_stride/inverse'})
conv_patch_stride = Const(
graph, {
'value': int64_array([node.patch_stride]),
'name': node_name + '/patch_stride/'
}).create_node()
pow_node.in_port(0).connect(conv_patch_stride.out_port(0))
mul = Mul(graph, {
'name': node_name + '/mul_inverse_stride_h'
}).create_node()
mul.in_port(0).connect(split.out_port(1))
mul.in_port(1).connect(pow_node.out_port(0))
concat = create_op_with_const_inputs(
graph, Concat, {2: int64_array([1])}, {
'name': node_name + '/concat_all_dims',
'in_ports_count': 4,
'axis': 0
})
concat.in_port(0).connect(split.out_port(0))
concat.in_port(1).connect(mul.out_port(0))
concat.in_port(3).connect(conv_patch_stride.out_port(0))
reshape_in = Reshape(graph, {
'name': node_name + '/reshape_in'
}).create_node()
reshape_in.in_port(1).connect(concat.out_port(0))
# create Reshape after Convolution
reshape_out = create_op_node_with_second_input(
graph, Reshape, int64_array([0, -1]),
{'name': node_name + '/reshape_out'})
# connect input_reshape_node
source = node.in_port(0).get_source()
node.in_port(0).get_connection().set_source(reshape_in.out_port(0))
reshape_in.in_port(0).connect(source)
# connect output_reshape_node
node.out_port(0).get_connection().set_source(reshape_out.out_port(0))
node.out_port(0).connect(reshape_out.in_port(0))
def _fuse_linear_sequence(graph: Graph, start_node: Node):
"""
This function finds the sequence of Mul/Add operations and replaces this sequence with two ops (Mul->Add).
:param graph:
:param start_node: The first operation of the sequence
"""
fnodes = [start_node]
while True:
node = fnodes[-1]
destinations = node.out_port(0).get_destinations()
if len(destinations) != 1:
break
dst_node = destinations[0].node
if dst_node.soft_get('op') in [
'Mul', 'Add'
] and get_value_in_port(dst_node) is not None and dst_node.soft_get(
'can_be_fused') is True:
fnodes.append(dst_node)
else:
break
if len(fnodes) == 1 or (len(fnodes) == 2 and fnodes[0].op == 'Mul'
and fnodes[1].op == 'Add'):
return False
input_shape = get_tensor_in_port(start_node).data.get_shape()
init_dims_cnt = len(
input_shape) - 2 if graph.graph['layout'] == 'NCHW' else 1
mul = np.ones([1 for x in range(init_dims_cnt)])
add = np.zeros([1 for x in range(init_dims_cnt)])
first_mul_name = None
first_add_name = None
for node in fnodes:
const_port_value = get_value_in_port(node).data.get_value()
if node.op == 'Mul':
if first_mul_name is None:
first_mul_name = node.name
mul = mul * const_port_value
add = add * const_port_value
elif node.op == 'Add':
if first_add_name is None:
first_add_name = node.name
add = add + const_port_value
# If mul is scalar we broadcast it to biases shape
if mul.shape != add.shape and len(mul.shape) == 1 and mul.shape[0] == 1:
mul = np.array([mul[0] for x in range(add.shape[0])])
assert (np.array_equal(
get_tensor_in_port(fnodes[0]).data.get_shape(),
fnodes[-1].out_port(0).data.get_shape()))
mul_op = Mul(graph,
dict(name='{}/Fused_Mul_'.format(first_mul_name or '')))
add_op = Add(graph,
dict(name='{}/Fused_Add_'.format(first_add_name or '')))
in_port = get_tensor_in_port(fnodes[0])
out_port = fnodes[-1].out_port(0)
"""
Four cases considered below:
1. Mul and Add have valid values (mul value != 1 and add value != 0)
2. Only Mul has valid values, so we add only Mul node
3. Only Add has valid values, so we add only Add node
4. When Mul and Add has not valid values we just merge two data nodes
"""
if any([x != 0
for x in np.nditer(add)]) and any([x != 1
for x in np.nditer(mul)]):
# Const\ Const\
# ----->Mul------>Add-->
mul_const = Const(graph, dict(name="data_mul_",
value=np.array(mul))).create_node()
add_const = Const(graph, dict(name="data_add_",
value=np.array(add))).create_node()
mul_node = mul_op.create_node()
add_node = add_op.create_node()
in_port.get_connection().set_destination(mul_node.in_port(0))
mul_const.out_port(0).connect(mul_node.in_port(1))
mul_node.out_port(0).connect(add_node.in_port(0))
add_const.out_port(0).connect(add_node.in_port(1))
out_port.get_connection().set_source(add_node.out_port(0))
elif any([x != 1 for x in np.nditer(mul)]):
# Const\
# ----->Mul-->
mul_const = Const(graph, dict(name="data_mul_",
value=np.array(mul))).create_node()
mul_node = mul_op.create_node()
in_port.get_connection().set_destination(mul_node.in_port(0))
mul_const.out_port(0).connect(mul_node.in_port(1))
out_port.get_connection().set_source(mul_node.out_port(0))
elif any([x != 0 for x in np.nditer(add)]):
# Const\
# ----->Add-->
add_const = Const(graph, dict(name="data_add_",
value=np.array(add))).create_node()
add_node = add_op.create_node()
in_port.get_connection().set_destination(add_node.in_port(0))
add_const.out_port(0).connect(add_node.in_port(1))
out_port.get_connection().set_source(add_node.out_port(0))
else:
source_node = in_port.get_source()
in_port.disconnect()
out_port.get_connection().set_source(source_node)
# Remove fused nodes
for node in fnodes:
graph.remove_node(node.id)
log.debug('Fused {} operations'.format(len(fnodes)))
return True
def replace_op(self, graph: Graph, node: Node):
input_node = node.in_node()
memory_pair_input = unique_id('id')
memory_pair_output = unique_id('id')
# Input -> FullyConnected
fc_layer_after_input_attrs = {
'name': 'input_fullyconnected',
'num_output': node.gifo_x_weights_shape[0],
'bias_term': True
}
embed_input(fc_layer_after_input_attrs, 1, 'weights',
node.gifo_x_weights)
embed_input(fc_layer_after_input_attrs, 2, 'biases', node.gifo_biases)
fc_layer_after_input = InnerProduct(
graph, fc_layer_after_input_attrs).create_node([input_node])
prev_lstm_output = Memory(
graph, {
'name': 'prev_memory_output',
'id': memory_pair_input,
'index': 1,
'size': 2,
'shape': np.array([node.gifo_r_weights_shape[1]],
dtype=np.int64)
}).create_node()
# *Memory(output) -> FullyConnected
fc_layer_from_prev_state_attrs = {
'name': 'prev_memory_output_fullyconnected',
'num_output': node.gifo_r_weights_shape[0],
'bias_term': False
}
embed_input(fc_layer_from_prev_state_attrs, 1, 'weights',
node.gifo_r_weights)
fc_layer_from_prev_state = InnerProduct(
graph,
fc_layer_from_prev_state_attrs).create_node([prev_lstm_output])
# Memory -> FullyConnected \
# *Eltwise(sum)
# Input -> FullyConnected /
join_input_prev_state_sum = Add(graph, {
'name': 'join_input_eltwise',
}).create_node([fc_layer_from_prev_state, fc_layer_after_input])
# *Eltwise(sum) -> Split
# it is split into 4 nodes: Act, Eltw*3
# the following order is mandatory
# ___Tanh
# /
# Split ---(2)Eltwise(sum)
# |\
# | \__(3)Eltwise(sum)
# |____(4)Eltwise(sum)
split_joined_input = Split(
graph, {
'name': 'join_input_split',
'axis': 1,
'num_split': 4,
'out_ports_count': 4,
}).create_node([join_input_prev_state_sum])
prev_lstm_state = Memory(
graph, {
'name':
'prev_memory_state',
'id':
memory_pair_output,
'index':
1,
'size':
2,
'shape':
np.array([node.input_gate_weights.shape[0]], dtype=np.int64)
}).create_node()
# *Memory(state) -> *ScaleShift(input)
state_input_scaleshift_attrs = {
'name': 'input_scaleshift',
'bias_term': False
}
embed_input(state_input_scaleshift_attrs, 1, 'weights',
node.input_gate_weights)
state_input_scaleshift = ScaleShiftOp(
graph, state_input_scaleshift_attrs).create_node([prev_lstm_state])
# *Memory(state) -> *ScaleShift(forget)
state_forget_scaleshift_attrs = {
'name': 'forget_scaleshift',
'bias_term': False
}
embed_input(state_forget_scaleshift_attrs, 1, 'weights',
node.forget_gate_weights)
state_forget_scaleshift = ScaleShiftOp(
graph,
state_forget_scaleshift_attrs).create_node([prev_lstm_state])
# Split \
# (2)Eltwise(sum)
# Memory(state) -> *ScaleShift(input) /
join_prev_lstm_input_joined_input_sum = Add(
graph, {
'name': 'join_prev_lstm_input_joined_input_eltwise',
}).create_node([(split_joined_input, 1), state_input_scaleshift])
# Split \
# (3)Eltwise(sum)
# Memory(state) -> *ScaleShift(forget) /
join_prev_lstm_input_joined_forget_sum = Add(
graph, {
'name': 'join_prev_lstm_input_joined_forget_sum',
}).create_node([(split_joined_input, 2), state_forget_scaleshift])
# Split -> Tanh
remember_tahn = Tanh(graph, {
'name': 'remember_tahnv'
}).create_node([(split_joined_input, 0)])
# Split -> (2)Eltwise(sum) -> *Sigmoid
remember_sigmoid = Sigmoid(graph, {
'name': 'remember_sigmoid'
}).create_node([join_prev_lstm_input_joined_input_sum])
# Split -> (3)Eltwise(sum) -> **Sigmoid
forget_sigmoid = Sigmoid(graph, {
'name': 'forget_sigmoid'
}).create_node([join_prev_lstm_input_joined_forget_sum])
# *Memory(state) \
# (6)Eltwise(mul)
# Split -> (3)Eltwise(sum) -> **Sigmoid /
join_forget_prev_state_mul = Mul(graph, {
'name': 'join_forget_prev_state_mul',
}).create_node([forget_sigmoid, prev_lstm_state])
# Split -> Tahn \
# (5)Eltwise(mul)
# Split -> (2)Eltwise(sum) -> *Sigmoid /
join_remember_candidates_mul = Mul(
graph, {
'name': 'join_remember_candidates_mul',
}).create_node([remember_tahn, remember_sigmoid])
# (5)Eltwise(mul) \
# (7)Eltwise(sum)
# (6)Eltwise(mul) /
join_forget_remember_sum = Add(graph, {
'name': 'join_forget_remember_sum',
}).create_node(
[join_forget_prev_state_mul, join_remember_candidates_mul])
# (7)Eltwise(sum) -> Clamp
join_forget_clamp = Clamp(
graph, {
'name': 'join_forget_clamp',
'max': node.clip_value,
'min': -node.clip_value
}).create_node([join_forget_remember_sum])
#
# Clamp -> (2)Memory(state)
next_lstm_state = Memory(
graph, {
'name':
'next_lstm_state',
'id':
memory_pair_output,
'index':
0,
'size':
2,
'shape':
np.array([node.input_gate_weights.shape[0]], dtype=np.int64)
}).create_node([join_forget_clamp])
Result(graph, {
'name': 'next_lstm_state_out'
}).create_node([next_lstm_state])
# Clamp -> (2)Tahn
state_filtered_tahn = Tanh(graph, {
'name': 'state_filtered_tahn'
}).create_node([join_forget_clamp])
# Clamp -> (2)ScaleShift
clamp_scaleshift_attrs = {
'name': 'clamp_scaleshift',
'bias_term': False
}
embed_input(clamp_scaleshift_attrs, 1, 'weights',
node.output_gate_weights)
clamp_scaleshift = ScaleShiftOp(
graph, clamp_scaleshift_attrs).create_node([join_forget_clamp])
# Split \
# (4)Eltwise(sum)
# Clamp -> (2)ScaleShift /
join_next_lstm_input_joined_input_sum = Add(
graph, {
'name': 'join_next_lstm_input_joined_input_sum',
}).create_node([(split_joined_input, 3), clamp_scaleshift])
# (4)Eltwise(sum) -> (3)Sigmoid
output_sigmoid = Sigmoid(graph, {
'name': 'output_sigmoid'
}).create_node([join_next_lstm_input_joined_input_sum])
# (4)Eltwise(sum) -> (3)Sigmoid \
# (5)Eltwise(mul)
# Clamp -> (2)Tahn /
joined_output_mul = Mul(graph, {
'name': 'joined_output_mul'
}).create_node([state_filtered_tahn, output_sigmoid])
# (5)Eltwise(mul) -> (3)FullyConnected
fc_output_attrs = {
'name': 'FullyConnected',
'num_output': node.projection_weights_shape[0],
'bias_term': False
}
embed_input(fc_output_attrs, 1, 'weights', node.projection_weights)
fc_output = InnerProduct(graph, fc_output_attrs).create_node(
[joined_output_mul])
# / (2)Memory(output)
# (3)FullyConnected
# \ Output (any next node) (edge created automatically after replacement)
next_lstm_output = Memory(
graph, {
'name': 'next_lstm_output',
'id': memory_pair_input,
'index': 0,
'size': 2,
'shape': np.array([node.gifo_r_weights_shape[1]],
dtype=np.int64)
}).create_node([fc_output])
Result(graph, {
'name': 'next_lstm_output_out'
}).create_node([next_lstm_output])
return [fc_output.id]
def extract(cls, node):
Mul.update_node_stat(
node, {'data_type': tf_dtype_extractor(node.pb.attr["T"].type)})
return cls.enabled
def replace_sub_graph(self, graph: Graph, match: Dict[str, Node]):
node = match['op']
name = node.name
min_port_tuple = (node.in_port(1).get_source().node,
node.in_port(1).get_source().idx)
max_port_tuple = (node.in_port(2).get_source().node,
node.in_port(2).get_source().idx)
node.in_port(1).disconnect()
node.in_port(2).disconnect()
# make sure min < max
min_less_max = Less(graph, {
'name': name + '/if_min_less_max'
}).create_node([min_port_tuple, max_port_tuple])
minimum = Select(graph, {
'name': name + '/minimum'
}).create_node([min_less_max, min_port_tuple, max_port_tuple])
maximum = Select(graph, {
'name': name + '/maximum'
}).create_node([min_less_max, max_port_tuple, min_port_tuple])
# to create zero of limits data type, we multiply it by integer zero
zero = create_op_node_with_second_input(graph,
Mul,
int64_array(0),
{'name': name + '/zero'},
input_node=minimum)
# if 0 < min < max: min_adj = 0 and max_adj = max - min
min_greater_zero = Greater(graph, {
'name': name + '/if_minimum_greater_zero'
}).create_node([minimum, zero])
max_minus_min = Sub(graph, {
'name': name + '/max_minus_min'
}).create_node([maximum, minimum])
minimum = Select(graph, {
'name': name + '/first_adj_min'
}).create_node([min_greater_zero, zero, minimum])
maximum = Select(graph, {
'name': name + '/first_adj_max'
}).create_node([min_greater_zero, max_minus_min, maximum])
# if min < max < 0: min_adj = min - max and max_adj = 0
max_less_zero = Less(graph, {
'name': name + '/if_max_less_zero'
}).create_node([maximum, zero])
min_minus_max = Sub(graph, {
'name': name + '/min_minus_max'
}).create_node([minimum, maximum])
minimum = Select(graph, {
'name': name + '/second_adj_min'
}).create_node([max_less_zero, min_minus_max, minimum])
maximum = Select(graph, {
'name': name + '/second_adj_max'
}).create_node([max_less_zero, zero, maximum])
# scale = (max - min) / (2 ^ num_bits - 1),
float_range = Sub(graph, {
'name': name + '/float_range'
}).create_node([maximum, minimum])
quant_min_value, quant_max_value = int(
node.narrow_range), 2**node.num_bits - 1
int_range = Const(
graph,
dict(name=name + '/int_range',
value=quant_max_value - quant_min_value)).create_node()
scale = Div(graph, {
'name': name + '/scale'
}).create_node([float_range, int_range])
# min_adj = scale * round(min / scale)
descaled_min = Div(graph, {
'name': name + '/descaled_min'
}).create_node([minimum, scale])
rounded_descaled_min = Round(graph, {
'name': name + '/rounded_descaled_min'
}).create_node([descaled_min])
min_adj = Mul(graph, {
'name': name + '/min_adj'
}).create_node([scale, rounded_descaled_min])
# max_adj = max + min_adj - min.
adjustment = Sub(graph, {
'name': name + '/limits_adjustment'
}).create_node([min_adj, minimum])
max_adj = Add(graph, {
'name': name + '/max_adj'
}).create_node([maximum, adjustment])
# FakeQuantize operation has 5 inputs instead of 3 inputs in TensorFlow
node.add_input_port(3, skip_if_exist=True)
node.add_input_port(4, skip_if_exist=True)
node.in_port(1).connect(min_adj.out_port(0))
node.in_port(2).connect(max_adj.out_port(0))
node.in_port(3).connect(min_adj.out_port(0))
node.in_port(4).connect(max_adj.out_port(0))
FakeQuantize.update_node_stat(node, {'levels': node['levels']})
