def find_and_replace_pattern(self, graph: Graph):
for dequantize_node in graph.get_op_nodes(op='DequantizeLinear'):
node_name = dequantize_node.soft_get('name', dequantize_node.id)
axis = dequantize_node.soft_get('axis', None)
scale_y_shape = dequantize_node.in_port(1).data.get_shape()
model_data_type = data_type_str_to_np(
graph.graph['cmd_params'].data_type)
cast = Cast(graph, {
'dst_type': model_data_type,
'name': node_name + '/Cast'
}).create_node()
dequantize_node.in_port(0).get_connection().set_destination(
cast.in_port(0))
mul = Mul(graph, {'can_be_fused': False}).create_node()
is_second_port_connected = dequantize_node.is_in_port_connected(2)
if is_second_port_connected:
# its is necessary not to replace subrtract for pattern in offline transformations
# See ConvertQuantizeDequantize transformation in ngraph
sub = Sub(graph, {
'name': node_name + '/Sub',
'zero_point_sub': True
}).create_node()
cast.out_port(0).connect(sub.in_port(0))
dequantize_node.in_port(2).get_connection().set_destination(
sub.in_port(1))
sub.out_port(0).connect(mul.in_port(0))
else:
cast.out_port(0).connect(mul.in_port(0))
dequantize_node.in_port(1).get_connection().set_destination(
mul.in_port(1))
dequantize_node.out_port(0).get_connection().set_source(
mul.out_port(0))
rename_nodes([(dequantize_node, node_name + '/TBD'),
(mul, node_name)])
assert scale_y_shape is not None
if axis is not None and len(
scale_y_shape) > 0 and scale_y_shape[0] > 1:
input_shape = cast.in_port(0).data.get_shape()
target_shape = np.ones(len(input_shape), np.int64)
target_shape[axis] = input_shape[axis]
mul_reshape = create_op_with_const_inputs(
graph, Reshape, {1: int64_array(target_shape)},
{'name': node_name + '/Reshape/Mul'})
mul.in_port(1).get_connection().set_destination(
mul_reshape.in_port(0))
mul_reshape.out_port(0).connect(mul.in_port(1))
if is_second_port_connected:
sub_reshape = create_op_with_const_inputs(
graph, Reshape, {1: int64_array(target_shape)},
{'name': node_name + '/Reshape/Sub'})
sub.in_port(1).get_connection().set_destination(
sub_reshape.in_port(0))
sub_reshape.out_port(0).connect(sub.in_port(1))
def calculate_prior_box_value(value: Node, value_to_div: Port,
value_to_add: Port):
"""
:param value: Node with value. Here is supposed the node with op='Split'
:param value_to_div: Output port with values to be divided by 2
:param value_to_add: Output port with values to be added to values from value_to_div port
:return: Sub and Add nodes
The sub-graph can be described by formulas:
min = value[value_to_add] - (value[value_to_div] / 2)
max = value[value_to_add] + (value[value_to_div] / 2)
"""
graph = value.graph
dtype = data_type_str_to_np(graph.graph['cmd_params'].data_type)
_min = Sub(graph, dict(name=value.name + '/Sub')).create_node()
div = create_op_node_with_second_input(graph,
Div,
mo_array([2], dtype=dtype),
op_attrs=dict(name=value.name +
'/Div'))
div.in_port(0).connect(value_to_div)
_min.in_port(0).connect(value_to_add)
_min.in_port(1).connect(div.out_port(0))
_max = Add(graph, dict(name=value.name + '/Add')).create_node()
_max.in_port(0).connect(div.out_port(0))
_max.in_port(1).connect(value_to_add)
return _min, _max
def find_and_replace_pattern(self, graph: Graph):
for node in graph.get_op_nodes(op='SpaceToBatch') + graph.get_op_nodes(
op='BatchToSpace'):
node.add_input_port(3, skip_if_exist=True)
# convert TF representation of the pads/crops as [N, 2] to IE representation: [N] and [N]
transposed_pads = create_op_with_const_inputs(
graph, Transpose, {1: int64_array([1, 0])})
node.in_port(2).get_connection().set_destination(
transposed_pads.in_port(0))
split_pads = create_op_with_const_inputs(graph, Split,
{1: int64_array(0)},
{'num_splits': 2})
transposed_pads.out_port(0).connect(split_pads.in_port(0))
for port_ind in range(2):
node.in_port(port_ind + 2).connect(
split_pads.out_port(port_ind))
node.in_port(port_ind + 2).get_connection().insert_node(
create_op_with_const_inputs(graph, Squeeze,
{1: int64_array([0])}))
# add zeros/ones to related inputs to align it with data input
in0_rank = Rank(graph, {
'name': node.name + '/rank_0'
}).create_node()
in1_shape = Shape(graph, {
'name': node.name + '/rank_1'
}).create_node()
diff_size = Sub(graph, {
'name': node.name + '/sub_0'
}).create_node()
diff = Sub(graph, {'name': node.name + '/sub_1'}).create_node()
const_begin = Const(graph, {
'value': int64_array([1])
}).create_node()
const_pad_val = Const(graph, {
'value': int64_array(1)
}).create_node()
block_shape = Pad(graph, {
'name': node.name + '/aligned_block_shape',
'mode': 'constant'
}).create_node()
# in case of SpaceToBatch begin = pads_begin, end = pads_end
# in case of BatchToSpace begin = crops_begin, end = crops_end
new_begin_name = '/aligned_pads_begin'
new_end_name = '/aligned_pads_end'
if node.type == 'BatchToSpace':
new_begin_name = '/aligned_crops_begin'
new_end_name = '/aligned_crops_end'
begin = Pad(graph, {
'name': node.name + new_begin_name,
'mode': 'constant'
}).create_node()
end = Pad(graph, {
'name': node.name + new_end_name,
'mode': 'constant'
}).create_node()
in0_rank_1d = create_op_node_with_second_input(
graph, Unsqueeze, int64_array([0]),
{'name': node.name + '/1d_rank_of_0'}, in0_rank)
node.in_port(0).get_source().connect(in0_rank.in_port(0))
node.in_port(1).get_source().connect(in1_shape.in_port(0))
in0_rank_1d.out_port(0).connect(diff_size.in_port(0))
in1_shape.out_port(0).connect(diff_size.in_port(1))
diff_size.out_port(0).connect(diff.in_port(0))
const_begin.out_port(0).connect(diff.in_port(1))
const_pad_val.out_port(0).connect(block_shape.in_port(3))
inputs_array = [block_shape, begin, end]
for idx, input_to_node in enumerate(inputs_array):
name_of_input_to_node = input_to_node.name
node.in_port(idx + 1).get_connection().set_destination(
input_to_node.in_port(0))
const_begin.out_port(0).connect(input_to_node.in_port(1))
diff.out_port(0).connect(input_to_node.in_port(2))
input_to_node.out_port(0).connect(node.in_port(idx + 1))
convert = Cast(graph, {
'name': name_of_input_to_node + '/i64',
'dst_type': np.int64
}).create_node()
input_to_node.in_port(0).get_connection().insert_node(convert)
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch):
reshape_classes_node = create_op_node_with_second_input(graph, Reshape, int64_array([0, -1]),
dict(name='do_reshape_classes'),
match.single_input_node(1)[0])
initial_priors_node = match.single_input_node(2)[0]
priors_name = initial_priors_node.soft_get('name', initial_priors_node.id)
# model calculates identical prior boxes for each batch, so we take first slice of them
begin = Const(graph, {'value': mo_array([0, 0, 0], dtype=np.int32)}).create_node()
end = Const(graph, {'value': mo_array([1, 0, 0], dtype=np.int32)}).create_node()
stride = Const(graph, {'value': mo_array([1, 1, 1], dtype=np.int32)}).create_node()
priors_node = StridedSlice(graph, {'name': priors_name + '/0_batch_slice',
'begin_mask': int64_array([1, 1, 1]),
'end_mask': int64_array([1, 0, 0]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])}).create_node()
initial_priors_node.out_port(0).connect(priors_node.in_port(0))
begin.out_port(0).connect(priors_node.in_port(1))
end.out_port(0).connect(priors_node.in_port(2))
stride.out_port(0).connect(priors_node.in_port(3))
placeholders = graph.get_op_nodes(type='Parameter')
assert len(placeholders) == 1, "{} replacer requires model to have one Placeholder, but current model has " \
"{} placeholders".format(self.replacement_id, len(placeholders))
placeholder = placeholders[0]
# scale prior boxes to the [0, 1] interval
node_with_scales_for_prior_boxes = self.placeholder_scales(placeholder)
priors_scale_node = Mul(graph, {'name': 'scale_priors'}).create_node()
broadcast = Broadcast(graph, {'name': 'scales_broadcast'}).create_node()
shape_of_priors = Shape(graph, {'name': 'priors_shape'}).create_node()
priors_node.out_port(0).connect(shape_of_priors.in_port(0))
broadcast.in_port(1).connect(shape_of_priors.out_port(0))
broadcast.in_port(0).connect(node_with_scales_for_prior_boxes.out_port(0))
priors_scale_node.in_port(0).connect(priors_node.out_port(0))
priors_scale_node.in_port(1).connect(broadcast.out_port(0))
try:
variance = match.custom_replacement_desc.custom_attributes['variance']
except:
raise Error('There is no variance attribute in {} replacement config file `custom_attributes`'
''.format(self.replacement_id))
priors = self.append_variances(priors_scale_node, variance)
# calculate prior boxes widths and heights
split_node = create_op_with_const_inputs(
graph, VariadicSplit, {1: int64_array(2), 2: int64_array([1, 1, 1, 1])}, {'out_ports_count': 4},
priors_scale_node)
priors_width_node = Sub(graph, dict(name=split_node.name + '/sub_2-0_')
).create_node([(split_node, 2), (split_node, 0)])
priors_height_node = Sub(graph, dict(name=split_node.name + '/sub_3-1_')
).create_node([(split_node, 3), (split_node, 1)])
# concat weights and heights into a single tensor and multiple with the box coordinates regression values
# WA with 3 Concats instead of 1 for keeping model reshapable
# concat_width_height_node = Concat(graph, {'name': 'concat_priors_width_height', 'axis': -1,
# 'in_ports_count': 4}).create_node(
# [priors_width_node, priors_height_node, priors_width_node, priors_height_node])
concat_1 = Concat(graph, {'name': 'concat_width_height',
'axis': -1, 'in_ports_count': 2}).create_node([priors_width_node, priors_height_node])
concat_2 = Concat(graph, {'name': 'concat_width_height_width',
'axis': -1, 'in_ports_count': 2}).create_node([concat_1, priors_width_node])
concat_width_height_node = Concat(graph, {'name': 'concat_priors_width_height', 'axis': -1, 'in_ports_count': 2}
).create_node([concat_2, priors_height_node])
applied_width_height_regressions_node = Mul(graph, {'name': 'final_regressions'}).create_node(
[concat_width_height_node, match.single_input_node(0)[0]])
# reshape to 2D tensor as Inference Engine Detection Output layer expects
reshape_regression_node = create_op_node_with_second_input(graph, Reshape, int64_array([0, -1]),
dict(name='reshape_regression'),
applied_width_height_regressions_node)
detection_output_op = DetectionOutput(graph, match.custom_replacement_desc.custom_attributes)
# get nms from the original network
iou_threshold = None
nms_nodes = graph.get_op_nodes(op='NonMaxSuppression')
if len(nms_nodes) > 0:
# it is highly unlikely that for different classes NMS has different
# moreover DetectionOutput accepts only scalar values for iou_threshold (nms_threshold)
iou_threshold = nms_nodes[0].in_node(3).value
if iou_threshold is None:
raise Error('During {} `iou_threshold` was not retrieved from RetinaNet graph'.format(self.replacement_id))
detection_output_node = detection_output_op.create_node(
[reshape_regression_node, reshape_classes_node, priors],
dict(name=detection_output_op.attrs['type'], nms_threshold=iou_threshold, clip_after_nms=1, normalized=1,
variance_encoded_in_target=0, background_label_id=1000))
# As outputs are replaced with a postprocessing node, outgoing tensor names are no longer
# correspond to original tensors and should be removed from output->Result edges
out_nodes = []
for out in range(match.outputs_count()):
out_nodes.append(match.output_node(out)[0])
clear_tensor_names_info(out_nodes)
return {'detection_output_node': detection_output_node}
def extract(cls, node: Node):
axis = onnx_attr(node, 'axis', 'i', default=None)
Sub.update_node_stat(node, {'axis': axis})
return cls.enabled
def extract(cls, node):
Sub.update_node_stat(node, {'data_type': tf_dtype_extractor(node.pb.attr["T"].type)})
return cls.enabled
def replace_sub_graph(self, graph: Graph, match: Dict[str, Node]):
node = match['op']
name = node.name
min_port_tuple = (node.in_port(1).get_source().node,
node.in_port(1).get_source().idx)
max_port_tuple = (node.in_port(2).get_source().node,
node.in_port(2).get_source().idx)
node.in_port(1).disconnect()
node.in_port(2).disconnect()
# make sure min < max
min_less_max = Less(graph, {
'name': name + '/if_min_less_max'
}).create_node([min_port_tuple, max_port_tuple])
minimum = Select(graph, {
'name': name + '/minimum'
}).create_node([min_less_max, min_port_tuple, max_port_tuple])
maximum = Select(graph, {
'name': name + '/maximum'
}).create_node([min_less_max, max_port_tuple, min_port_tuple])
# to create zero of limits data type, we multiply it by integer zero
zero = create_op_node_with_second_input(graph,
Mul,
int64_array(0),
{'name': name + '/zero'},
input_node=minimum)
# if 0 < min < max: min_adj = 0 and max_adj = max - min
min_greater_zero = Greater(graph, {
'name': name + '/if_minimum_greater_zero'
}).create_node([minimum, zero])
max_minus_min = Sub(graph, {
'name': name + '/max_minus_min'
}).create_node([maximum, minimum])
minimum = Select(graph, {
'name': name + '/first_adj_min'
}).create_node([min_greater_zero, zero, minimum])
maximum = Select(graph, {
'name': name + '/first_adj_max'
}).create_node([min_greater_zero, max_minus_min, maximum])
# if min < max < 0: min_adj = min - max and max_adj = 0
max_less_zero = Less(graph, {
'name': name + '/if_max_less_zero'
}).create_node([maximum, zero])
min_minus_max = Sub(graph, {
'name': name + '/min_minus_max'
}).create_node([minimum, maximum])
minimum = Select(graph, {
'name': name + '/second_adj_min'
}).create_node([max_less_zero, min_minus_max, minimum])
maximum = Select(graph, {
'name': name + '/second_adj_max'
}).create_node([max_less_zero, zero, maximum])
# scale = (max - min) / (2 ^ num_bits - 1),
float_range = Sub(graph, {
'name': name + '/float_range'
}).create_node([maximum, minimum])
quant_min_value, quant_max_value = int(
node.narrow_range), 2**node.num_bits - 1
int_range = Const(
graph,
dict(name=name + '/int_range',
value=quant_max_value - quant_min_value)).create_node()
scale = Div(graph, {
'name': name + '/scale'
}).create_node([float_range, int_range])
# min_adj = scale * round(min / scale)
descaled_min = Div(graph, {
'name': name + '/descaled_min'
}).create_node([minimum, scale])
rounded_descaled_min = Round(graph, {
'name': name + '/rounded_descaled_min'
}).create_node([descaled_min])
min_adj = Mul(graph, {
'name': name + '/min_adj'
}).create_node([scale, rounded_descaled_min])
# max_adj = max + min_adj - min.
adjustment = Sub(graph, {
'name': name + '/limits_adjustment'
}).create_node([min_adj, minimum])
max_adj = Add(graph, {
'name': name + '/max_adj'
}).create_node([maximum, adjustment])
# FakeQuantize operation has 5 inputs instead of 3 inputs in TensorFlow
node.add_input_port(3, skip_if_exist=True)
node.add_input_port(4, skip_if_exist=True)
node.in_port(1).connect(min_adj.out_port(0))
node.in_port(2).connect(max_adj.out_port(0))
node.in_port(3).connect(min_adj.out_port(0))
node.in_port(4).connect(max_adj.out_port(0))
FakeQuantize.update_node_stat(node, {'levels': node['levels']})
def dequantize_data(fake_quantize: Node, dst_type: type,
quantized_type: type) -> Node:
graph = fake_quantize.graph
quantized_data = fake_quantize.in_port(0).get_source().node
name = fake_quantize.soft_get('name', fake_quantize.id)
assert quantized_data.soft_get('type') == 'Convert' and quantized_data.dst_type == quantized_type, \
'Weights aren`t compressed as expected for node {}'.format(fake_quantize.soft_get('name', fake_quantize.id))
dequantizing_cast = Cast(
graph,
dict(name=quantized_data.name +
"/to_{}".format(np_data_type_to_destination_type(dst_type)),
dst_type=dst_type,
stop_value_propagation=True)).create_node()
fake_quantize.in_port(0).get_connection().set_destination(
dequantizing_cast.in_port(0))
# limits of dequantize
in_low = fake_quantize.in_port(1).get_source()
in_high = fake_quantize.in_port(2).get_source()
out_low = fake_quantize.in_port(3).get_source()
out_high = fake_quantize.in_port(4).get_source()
# scale calculation
output_range = Sub(graph, {
'name': name + '/output_range'
}).create_node()
output_range.in_port(0).connect(out_high)
output_range.in_port(1).connect(out_low)
input_range = Sub(graph, {'name': name + '/input_range'}).create_node()
input_range.in_port(0).connect(in_high)
input_range.in_port(1).connect(in_low)
scale = Div(graph, {'name': name + '/scale'}).create_node()
scale.in_port(0).connect(output_range.out_port(0))
scale.in_port(1).connect(input_range.out_port(0))
# shift calculation
descaled_output_low = Div(graph, {
'name': name + '/descaled_output_low'
}).create_node()
descaled_output_low.in_port(0).connect(out_low)
descaled_output_low.in_port(1).connect(scale.out_port(0))
shift = Sub(graph, {'name': name + '/shift'}).create_node()
shift.in_port(0).connect(in_low)
shift.in_port(1).connect(descaled_output_low.out_port(0))
zero = Const(graph, {
'name': name + '/zero',
'value': mo_array(0, dtype=dst_type)
}).create_node()
scale_eq_zero = Equal(graph, {
'name': name + '/scale_eq_zero'
}).create_node()
scale_eq_zero.in_port(0).connect(scale.out_port(0))
scale_eq_zero.in_port(1).connect(zero.out_port(0))
zero_point = Select(graph, {
'name': name + '/zero_point'
}).create_node()
zero_point.in_port(0).connect(scale_eq_zero.out_port(0))
zero_point.in_port(1).connect(zero.out_port(0))
zero_point.in_port(2).connect(shift.out_port(0))
# DeQuantize(x) == Mul(Sub(x, zero_point), scale)
sub_zp = Sub(graph, {'name': name + '/minus_zp'}).create_node()
sub_zp.in_port(0).connect(dequantizing_cast.out_port(0))
sub_zp.in_port(1).connect(zero_point.out_port(0))
mul_scale = Mul(graph, {
'name': name + '/mulpiply_by_scale'
}).create_node()
mul_scale.in_port(0).connect(sub_zp.out_port(0))
mul_scale.in_port(1).connect(scale.out_port(0))
fake_quantize.out_port(0).get_connection().set_source(
mul_scale.out_port(0))
graph.remove_nodes_from([fake_quantize.id, fake_quantize.out_node(0)])
def mxrepeat_decomposition(node: Node):
graph = node.graph
name = node.soft_get('name', node.id)
rename_node(node, name + '/to_be_removed')
# Unqueeze
input_rank = Rank(graph, {'name': name + '/Rank'}).create_node()
node.in_port(0).get_source().connect(input_rank.in_port(0))
axis = get_canonical_axis_index_node(input_rank, node.axis)
unsqueeze_axis = create_op_node_with_second_input(
graph,
Add,
int64_array([1]), {'name': name + '/Unsqueeze/Axis'},
input_node=axis)
unsqueeze = Unsqueeze(graph, {
'name': name + '/Unsqueeze'
}).create_node()
unsqueeze.in_port(1).connect(unsqueeze_axis.out_port(0))
# Tile (1, 1, ..., repeats, ..., 1)
# we generate tile array according to the following table:
# parts: | first | repeats | second |
# i: | 0, 1, ..., axis,| axis + 1,| ..., rank+1 |
# tile_array: | 1, 1, ..., 1 ,| repeats ,| ..., 1 |
one = Const(graph, {
'name': name + '/Broadcast/One',
'value': int64_array([1])
}).create_node()
first_ones = Broadcast(graph, {
'name': name + '/Broadcast/Ones_first_part'
}).create_node()
first_ones.in_port(0).connect(one.out_port(0))
first_ones.in_port(1).connect(unsqueeze_axis.out_port(0))
repeats = Const(graph, {
'name': name + '/repeats',
'value': int64_array([node.repeats])
}).create_node()
second_ones = Broadcast(graph, {
'name': name + '/Broadcast/Ones_second_part'
}).create_node()
second_part_broadcast_shape = Sub(
graph, {
'name': name + '/Broadcast/Shape/second_part'
}).create_node()
second_part_broadcast_shape.in_port(0).connect(input_rank.out_port(0))
second_part_broadcast_shape.in_port(1).connect(
unsqueeze_axis.out_port(0))
second_ones.in_port(0).connect(one.out_port(0))
second_ones.in_port(1).connect(second_part_broadcast_shape.out_port(0))
tile_repeats = new_shape_node_from_shape_nodes(
[first_ones, repeats, second_ones])
tile = Tile(graph, {'name': name + '/Tile'}).create_node()
tile.in_port(1).connect(tile_repeats.out_port(0))
# Reshape (input_shape[:axis], input_shape[axis] * repeats, input_shape[axis+1:])
# we generate reshape dim array according to the following table:
# parts: | first | rep | second |
# i: | 0, 1, ... ,| axis, | ..., rank |
# dim_array: | inp_sh[i] ,| input_shape[axis] * repeats ,| inp_sh[i] |
input_shape = Shape(graph, {'name': name + '/Shape'}).create_node()
node.in_port(0).get_source().connect(input_shape.in_port(0))
first_input_shape_part = get_shape_values_by_range_idxs(
input_shape,
input_rank,
begin=0,
end=node.axis,
include_begin=True,
include_end=False)
original_axis_dim = create_op_with_const_inputs(
graph,
Gather, {2: int64_array(0)}, {'name': name + '/OriginalDim'},
input_node=input_shape)
original_axis_dim.in_port(1).connect(axis.out_port(0))
repeated_dimention = Mul(graph, {
'name': name + '/RepeatedDim'
}).create_node()
repeated_dimention.in_port(0).connect(original_axis_dim.out_port(0))
repeated_dimention.in_port(1).connect(repeats.out_port(0))
second_input_shape_part = get_shape_values_by_range_idxs(
input_shape,
input_rank,
begin=node.axis,
end=-1,
include_begin=False,
include_end=True)
output_shape = new_shape_node_from_shape_nodes([
first_input_shape_part, repeated_dimention, second_input_shape_part
])
reshape = Reshape(graph, {'name': name}).create_node()
rename_node(reshape, name)
reshape.in_port(1).connect(output_shape.out_port(0))
# Final connections
node.in_port(0).get_connection().set_destination(unsqueeze.in_port(0))
tile.in_port(0).connect(unsqueeze.out_port(0))
reshape.in_port(0).connect(tile.out_port(0))
node.out_port(0).get_connection().set_source(reshape.out_port(0))
def extract(cls, node):
Sub.update_node_stat(node)
return cls.enabled