def test_select_infer_condition_with_value(self, else_data_shape, than_data_shape, select_output_shape,
condition_value, else_value, than_value, output_value):
graph = build_graph_with_attrs(nodes_with_attrs=self.nodes, edges_with_attrs=self.edges,
update_nodes_attributes=[('condition_data', {'shape': np.array(select_output_shape),
'value': condition_value}),
('else_data', {'shape': np.array(else_data_shape),
'value': else_value}),
('than_data', {'shape': np.array(than_data_shape),
'value': than_value}),
('select_output',
{'shape': np.array(select_output_shape),
'value': None})
])
graph_ref = build_graph_with_attrs(nodes_with_attrs=self.nodes, edges_with_attrs=self.edges,
update_nodes_attributes=[
('condition_data', {'shape': np.array(else_data_shape),
'value': condition_value}),
('else_data', {'shape': np.array(else_data_shape),
'value': else_value}),
('than_data', {'shape': np.array(than_data_shape),
'value': than_value}),
('select_output', {'shape': np.array(select_output_shape),
'value': output_value})])
node = Node(graph, 'select')
Select.infer(node)
if else_value is not None and than_value is not None:
(flag, resp) = compare_graphs(graph, graph_ref, 'select_output', check_op_attrs=True)
self.assertTrue(flag, resp)
self.assertTrue(np.array_equal(graph.nodes['select_output']['value'], graph_ref.nodes['select_output']['value']))
def test_select_infer_condition_true(self):
graph = build_graph_with_attrs(nodes_with_attrs=self.nodes,
edges_with_attrs=self.edges,
update_nodes_attributes=[
('condition', {
'value': np.array([True])
}),
('select_output', {
'shape': np.array([2, 2]),
'value': np.ones((2, 2))
})
])
# We should propagate shapes and values
graph_ref = build_graph_with_attrs(nodes_with_attrs=self.nodes,
edges_with_attrs=self.edges,
update_nodes_attributes=[
('select_output', {
'shape': np.array([2, 2]),
'value': np.ones((2, 2))
})
])
tested_class = Select(graph=graph, attrs={})
node = Node(graph, 'select')
tested_class.infer(node)
(flag, resp) = compare_graphs(graph,
graph_ref,
'select_output',
check_op_attrs=True)
self.assertTrue(flag, resp)
def replace_pattern(self, graph: Graph, match: dict):
condition = match['cond_data']
true_value = match['Switch_2_input']
false_value = match['identity_data']
select = Select(
graph, dict(name=match['Merge'].name + '/Select/',
format='tf')).create_node(
inputs=[condition, true_value, false_value])
match['Merge'].out_port(0).get_connection().set_source(
select.out_port(0))
# Reconnect inputs to some_op
op = match['some_op']
assert 1 in op.in_ports() and 0 in op.in_ports()
op.in_port(0).disconnect()
op.in_port(1).disconnect()
match['Switch'].in_port(0).get_connection().set_destination(
op.in_port(0))
match['Switch_1'].in_port(0).get_connection().set_destination(
op.in_port(1))
graph.remove_nodes_from(nodes=[
match['Switch_1'].id, match['Switch'].id, match['Switch_2'].id,
match['Merge'].id
])
def test_select_infer_condition_shapes_broadcast(self, else_data_shape, than_data_shape, select_output_shape):
graph = build_graph_with_attrs(nodes_with_attrs=self.nodes, edges_with_attrs=self.edges,
update_nodes_attributes=[('else_data', {'shape': np.array(else_data_shape),
'value': np.zeros(else_data_shape, dtype=np.float)}),
('than_data', {'shape': np.array(than_data_shape),
'value': np.zeros(than_data_shape, dtype=np.float)}),
('select_output', {'shape': np.array(select_output_shape),
'value': np.zeros(select_output_shape, dtype=np.float)})
])
# We should propagate shapes and values
graph_ref = build_graph_with_attrs(nodes_with_attrs=self.nodes, edges_with_attrs=self.edges,
update_nodes_attributes=[
('else_data', {'shape': np.array(else_data_shape),
'value': np.zeros(else_data_shape, dtype=np.float)}),
('than_data', {'shape': np.array(than_data_shape),
'value': np.zeros(than_data_shape, dtype=np.float)}),
('select_output', {'shape': np.array(select_output_shape), 'value': np.zeros(select_output_shape)})])
tested_class = Select(graph=graph, attrs={})
node = Node(graph, 'select')
tested_class.infer(node)
(flag, resp) = compare_graphs(graph, graph_ref, 'select_output', check_op_attrs=True)
self.assertTrue(flag, resp)
def test_select_infer_assert_shapes(self):
graph = build_graph_with_attrs(nodes_with_attrs=self.nodes, edges_with_attrs=self.edges,
update_nodes_attributes=[('else_data', {'shape': np.array([3, 3]), 'value':np.zeros((3, 3))})])
tested_class = Select(graph=graph, attrs={})
node = Node(graph, 'select')
with self.assertRaisesRegex(AssertionError, "Input shape do not broadcast"):
tested_class.infer(node)
def test_select_infer_assert_shapes(self):
graph = build_graph_with_attrs(nodes_with_attrs=self.nodes, edges_with_attrs=self.edges,
update_nodes_attributes=[('else', {'shape': np.array([3,3]), 'value':np.zeros((3,3))})])
tested_class = Select(graph=graph, attrs={})
node = Node(graph, 'select')
with self.assertRaisesRegex(AssertionError, "TensorFlow \'Select\' operation has 3 inputs: \'condition\',"
" \'then\' and \'else\' tensors.\'then\' and \'else\' tensors"
" must have the same shape by TensorFlow reference"):
tested_class.infer(node)
def build_select_graph_and_infer(condition_value,
then_value,
else_value,
out_value,
condition_shape=None,
then_shape=None,
else_shape=None,
out_shape=None,
auto_broadcast='numpy',
fw_format=None):
if then_value is not None:
then_shape = int64_array(then_value.shape)
if else_value is not None:
else_shape = int64_array(else_value.shape)
nodes = {
**valued_const_with_data('then', then_value, then_shape),
**valued_const_with_data('else', else_value, else_shape),
**valued_const_with_data('condition', condition_value, condition_shape),
**regular_op_with_empty_data(
'select', {
'op': 'Select',
'auto_broadcast': auto_broadcast,
'format': fw_format
}),
**result('out'),
}
edges = [
*connect('condition', '0:select'),
*connect('then', '1:select'),
*connect('else', '2:select'),
*connect('select', 'out'),
]
graph = build_graph(nodes, edges)
select_node = Node(graph, 'select')
Select.infer(select_node)
select_out_node = Node(graph, 'select_d')
value_desc = 'values'
ref_val = out_value
actual_val = select_out_node['value']
if out_shape is not None:
value_desc = 'shapes'
ref_val = out_shape
actual_val = select_out_node['shape']
assert select_out_node[
'value'] is None, "if 'out_shape' is defined manually 'value' must be None"
flag = strict_compare_tensors(actual_val, ref_val)
msg = '' if flag else 'reference {} and actual {} {} do not match\n'.format(
ref_val, actual_val, value_desc)
return flag, msg
def test_select_infer_assert_condition_bool(self):
graph = build_graph_with_attrs(nodes_with_attrs=self.nodes, edges_with_attrs=self.edges,
update_nodes_attributes=[('condition', {'value': np.array([3])})])
tested_class = Select(graph=graph, attrs={})
node = Node(graph, 'select')
with self.assertRaisesRegex(AssertionError, "TensorFlow \'Select\' operation has 3 inputs: \'condition\',"
" \'then\' and \'else\' tensors. Value of \'condition\' tensor"
" must be boolen by TensorFlow reference"):
tested_class.infer(node)
def find_and_replace_pattern(self, graph: Graph):
for merge in graph.get_op_nodes(op='Merge'):
for merge_switch_in_port in range(2):
if merge.in_port(merge_switch_in_port).disconnected() or \
merge.in_port(merge_switch_in_port).get_source().node.op != 'Switch':
continue
switch_2 = merge.in_port(
merge_switch_in_port).get_source().node
if merge.in_port(1 - merge_switch_in_port).disconnected() or \
merge.in_port(1 - merge_switch_in_port).get_source().node.op != 'Identity':
continue
false_value_port = merge.in_port(
1 - merge_switch_in_port).get_source()
true_value_port = switch_2.in_port(0).get_source()
op = false_value_port.node.in_port(0).get_source().node
if op.in_port(0).disconnected(
) or op.in_port(0).get_source().node.op != 'Switch':
continue
switch = op.in_port(0).get_source().node
if op.in_port(1).disconnected(
) or op.in_port(1).get_source().node.op != 'Switch':
continue
switch_1 = op.in_port(1).get_source().node
if switch.in_port(1).get_source() == switch_1.in_port(1).get_source() and \
switch.in_port(1).get_source() == switch_2.in_port(1).get_source():
select = Select(
graph,
dict(name=merge.soft_get('name') + '/Select/',
format='tf')).create_node()
select.in_port(0).connect(switch.in_port(1).get_source())
select.in_port(1).connect(true_value_port)
select.in_port(2).connect(false_value_port)
merge.out_port(0).get_connection().set_source(
select.out_port(0))
assert 1 in op.in_ports() and 0 in op.in_ports()
op.in_port(0).disconnect()
op.in_port(1).disconnect()
switch.in_port(0).get_connection().set_destination(
op.in_port(0))
switch_1.in_port(0).get_connection().set_destination(
op.in_port(1))
graph.remove_nodes_from(
nodes=[switch_1.id, switch.id, switch_2.id, merge.id])
# need to exit from the inner for loop because the Merge op has been removed
break
def replace_sub_graph(self, graph: Graph, match: dict):
tf_slice_node = match['op']
slice_name = tf_slice_node.soft_get('name', tf_slice_node.id)
slice_node = Slice(graph).create_node()
rename_nodes([(tf_slice_node, slice_name + '/to_be_removed'),
(slice_node, slice_name)])
ends_node = Add(graph, {'name': slice_name + '/ends'}).create_node()
# reconnect input, begin, and size from TFSlice to the subgraph with Slice
tf_slice_node.in_port(0).get_connection().set_destination(
slice_node.in_port(0))
tf_slice_node.in_port(1).get_connection().set_destination(
slice_node.in_port(1))
tf_slice_node.in_port(2).get_connection().set_destination(
ends_node.in_port(0))
slice_node.in_port(1).get_connection().add_destination(
ends_node.in_port(1))
max_ends = Shape(graph, {
'name': slice_name + '/ShapeOf'
}).create_node()
slice_node.in_port(0).get_connection().add_destination(
max_ends.in_port(0))
# check if size[i] == -1, will be applied elementwisely: len(size) = len(begin) = input_rank
where_max_ends_is_needed = create_op_with_const_inputs(
graph, Equal, {0: int64_array(-1)},
{'name': slice_name + '/where_max_ends_is_needed'})
ends_node.in_port(0).get_connection().add_destination(
where_max_ends_is_needed.in_port(1))
# select requires equal dtypes, need to convert ends to I64
ends_casted_to_i64 = Cast(graph, {
'name': slice_name + '/CastToI64',
'dst_type': np.int64
}).create_node([ends_node])
# if size[i] == 1 then take max_ends values
correct_ends = Select(graph, {
'name': slice_name + '/chosen_ends'
}).create_node(
[where_max_ends_is_needed, max_ends, ends_casted_to_i64])
correct_ends.out_port(0).connect(slice_node.in_port(2))
tf_slice_node.out_port(0).get_connection().set_source(
slice_node.out_port(0))
def replace_sub_graph(self, graph: Graph, match: dict):
node = match['op']
slice_name = node.soft_get('name', node.id)
slice_node = Slice(graph).create_node()
rename_nodes([(node, slice_name + '/to_be_removed'), (slice_node, slice_name)])
eq_node = Equal(graph, {'name': slice_name + '/equal'}).create_node()
minus_one_node = Const(graph, {'name': slice_name + '/minus_one', 'value': np.array(-1)}).create_node()
int32_max_node = Const(graph, {'name': slice_name + '/int32_max', 'value': np.iinfo(np.int32).max}).create_node()
select_node = Select(graph, {'name': slice_name + '/select'}).create_node()
# node to convert sizes to ends
sum_node = Add(graph, {'name': slice_name + '/end_const'}).create_node()
# reconnect input from tfslice to slice
node.in_port(0).get_source().connect(slice_node.in_port(0))
node.in_port(0).disconnect()
# reconnect begin of tfslice to start of slice
node.in_port(1).get_source().connect(slice_node.in_port(1))
node.in_port(1).disconnect()
# (size -> ends) reconnect begins and sizes to sum to evaluate ends for Slice
# connects begins to slice
slice_node.in_port(1).get_source().connect(sum_node.in_port(0))
node.in_port(2).get_source().connect(sum_node.in_port(1))
node.in_port(2).disconnect()
# if size[i] == -1 when take int32_max as end[i]
sum_node.in_port(1).get_source().connect(eq_node.in_port(0))
minus_one_node.out_port(0).connect(eq_node.in_port(1))
# from equal to 0 port of select
eq_node.out_port(0).connect(select_node.in_port(0))
# from int32_max to 1 of select
int32_max_node.out_port(0).connect(select_node.in_port(1))
# from sum to 2nd of select
sum_node.out_port(0).connect(select_node.in_port(2))
# out of select to end (2nd of slice)
select_node.out_port(0).connect(slice_node.in_port(2))
cast = Cast(graph, dict(name=sum_node.name + '/CastToI64', dst_type=np.int64)).create_node()
select_node.in_port(2).get_connection().insert_node(cast)
node.out_port(0).get_connection().set_source(slice_node.out_port(0))
def replace_sub_graph(self, graph: Graph, match: Dict[str, Node]):
node = match['op']
name = node.name
# Zero Point Nudging : Scale counting
f_min = node.in_port(1).get_source()
node.in_port(1).disconnect()
f_max = node.in_port(2).get_source()
node.in_port(2).disconnect()
f_diff = Sub(graph, {'name': name + '/float_range'}).create_node()
f_max.connect(f_diff.in_port(0))
f_min.connect(f_diff.in_port(1))
quant_min_value = int(node.narrow_range)
quant_max_value = 2 ** node.num_bits - 1
i_diff = Const(graph, dict(name=name + '/int_range', value=quant_max_value - quant_min_value)).create_node()
scale = Div(graph, {'name': name + '/scale'}).create_node()
f_diff.out_port(0).connect(scale.in_port(0))
i_diff.out_port(0).connect(scale.in_port(1))
# Zero Point Nudging : ZP from min counting
descaled_min = Div(graph, {'name': name + '/descaled_min'}).create_node()
f_min.connect(descaled_min.in_port(0))
scale.out_port(0).connect(descaled_min.in_port(1))
zero_point_from_min = Sub(graph, {'name': name + '/zero_point_from_min'}).create_node()
quant_min = Const(graph, {'value': quant_min_value, 'name': name + '/quant_min'}).create_node()
quant_min.out_port(0).connect(zero_point_from_min.in_port(0))
descaled_min.out_port(0).connect(zero_point_from_min.in_port(1))
# Zero Point Nudging : Nudged Zero Point counting
zp_lesser_q_mi = Less(graph, {'name': name + '/zero_point_from_min_less_quant_min'}).create_node()
zero_point_from_min.out_port(0).connect(zp_lesser_q_mi.in_port(0))
quant_min.out_port(0).connect(zp_lesser_q_mi.in_port(1))
zp_greater_q_ma = Greater(graph, {'name': name + '/zero_point_from_min_greater_quant_max'}).create_node()
zero_point_from_min.out_port(0).connect(zp_greater_q_ma.in_port(0))
quant_max = Const(graph, {'value': quant_max_value, 'name': name + '/quant_max'}).create_node()
quant_max.out_port(0).connect(zp_greater_q_ma.in_port(1))
rounded_zero_point_from_min = Round(graph, {'name': name + '/zero_point_from_min_rounding'}).create_node()
zero_point_from_min.out_port(0).connect(rounded_zero_point_from_min.in_port(0))
nudged_zero_point = Select(graph, {'name': name + '/nudging_zp_1_select_less_condition'}).create_node()
greater_condition = Select(graph, {'name': name + '/nudging_zp_2_select_greater_condition'}).create_node()
greater_condition.in_port(0).connect(zp_greater_q_ma.out_port(0))
greater_condition.in_port(1).connect(quant_max.out_port(0))
greater_condition.in_port(2).connect(rounded_zero_point_from_min.out_port(0))
nudged_zero_point.in_port(0).connect(zp_lesser_q_mi.out_port(0))
nudged_zero_point.in_port(1).connect(quant_max.out_port(0))
nudged_zero_point.in_port(2).connect(greater_condition.out_port(0))
nudged_i_min = Sub(graph, {'name': name + '/nudged_i_min'}).create_node()
quant_min.out_port(0).connect(nudged_i_min.in_port(0))
nudged_zero_point.out_port(0).connect(nudged_i_min.in_port(1))
nudged_i_max = Sub(graph, {'name': name + '/nudged_i_max'}).create_node()
quant_max.out_port(0).connect(nudged_i_max.in_port(0))
nudged_zero_point.out_port(0).connect(nudged_i_max.in_port(1))
nudged_min = Mul(graph, {'name': name + '/nudged_min'}).create_node()
nudged_i_min.out_port(0).connect(nudged_min.in_port(0))
scale.out_port(0).connect(nudged_min.in_port(1))
nudged_max = Mul(graph, {'name': name + '/nudged_max'}).create_node()
nudged_i_max.out_port(0).connect(nudged_max.in_port(0))
scale.out_port(0).connect(nudged_max.in_port(1))
nudged_min.out_port(0).connect(node.in_port(1))
nudged_max.out_port(0).connect(node.in_port(2))
# FakeQuantize operation has 5 inputs instead of 3 inputs in TensorFlow
node.add_input_port(3, skip_if_exist=True)
node.add_input_port(4, skip_if_exist=True)
node.in_port(3).connect(nudged_min.out_port(0))
node.in_port(4).connect(nudged_max.out_port(0))
FakeQuantize.update_node_stat(node, {'levels': node['levels']})
def insert_select(graph: Graph, node: Node):
context_len = node.frame_time + 1
if context_len == 1:
return
in_node_port = node.in_port(0).get_source()
in_node_shape = node.in_port(0).data.get_shape()
node.in_port(0).disconnect()
# add Select before saving state to avoid saving garbage
select_node = Select(graph, {'name': 'select_' + node.name}).create_node()
zero_else = create_const_with_batch_from_input(in_node_port, in_node_shape[1])
select_node.in_port(1).connect(in_node_port)
select_node.in_port(2).connect(zero_else.out_port(0))
# check if we have already appropriate iteration counter
existing_counters = find_pattern_matches(graph, nodes=[('mem_in', dict(op='ReadValue')),
('mem_in_data', dict(shape=int64_array([context_len]))),
('crop_mem_in', dict(op='Crop', axis=int64_array([1]),
offset=int64_array([1]),
dim=int64_array([context_len - 1]))),
('crop_mem_in_data', dict()),
('concat', dict(op='Concat', axis=1)),
('concat_data', dict()),
('const_1', dict(op='Const')),
('const_1_data', dict()),
('mem_out', dict(op='Assign')),
('crop_out', dict(op='Crop', axis=int64_array([1]),
offset=int64_array([0]),
dim=int64_array([1]))),
('crop_out_data', dict()),
('select', dict(op='Select'))
],
edges=[('mem_in', 'mem_in_data'), ('mem_in_data', 'crop_mem_in'),
('crop_mem_in', 'crop_mem_in_data'),
('crop_mem_in_data', 'concat', {'in': 0}),
('const_1', 'const_1_data'),
('const_1_data', 'concat', {'in': 1}),
('concat', 'concat_data'), ('concat_data', 'mem_out'),
('concat_data', 'crop_out'), ('crop_out', 'crop_out_data'),
('crop_out_data', 'select')])
counter_match = next(existing_counters, None)
if counter_match is not None:
ones = Node(graph, inverse_dict(counter_match)['const_1'])
input_port = Node(graph, inverse_dict(counter_match)['crop_out']).out_port(0)
else:
init_value_mem_out = create_const_with_batch_from_input(in_node_port, context_len, precision=np.int32)
mem_out = ReadValue(graph, {'name': 'iteration_number',
'variable_id': 'iteration_' + node.name}).create_node()
mem_out.in_port(0).connect(init_value_mem_out.out_port(0))
cut_first = Crop(graph, {'name': 'cut_first', 'axis': int64_array([1]),
'offset': int64_array([1]), 'dim': int64_array([context_len - 1])}).create_node()
cut_first.in_port(0).connect(mem_out.out_port(0))
ones = create_const_with_batch_from_input(in_node_port, 1, 1, np.int32)
concat = Concat(graph, {'name': 'concat_ones', 'in_ports_count': 2, 'axis': 1}).create_node()
concat.in_port(0).connect(cut_first.out_port(0))
concat.in_port(1).connect(ones.out_port(0))
mem_in = Assign(graph, {'name': 'iteration_number_out',
'variable_id': 'iteration_' + node.name}).create_node()
mem_in.in_port(0).connect(concat.out_port(0))
res = Result(graph, {}).create_node()
mem_in.out_port(0).connect(res.in_port(0))
cut_last = Crop(graph, {'name': 'cut_last', 'axis': int64_array([1]),
'offset': int64_array([0]), 'dim': int64_array([1])}).create_node()
cut_last.in_port(0).connect(concat.out_port(0))
input_port = cut_last.out_port(0)
# Check if data from memory is 1
# if it is True, we have correct data and should proceed with saving it to memory
# else we have not gathered context and have garbage here, shouldn't change initial state of memory
cast_in = Equal(graph, {'name': input_port.node.name + '/cast_to_bool'}).create_node()
cast_in.in_port(0).connect(ones.out_port(0))
cast_in.in_port(1).connect(input_port)
select_node.in_port(0).connect(cast_in.out_port(0))
select_node.out_port(0).connect(node.in_port(0))
select_node.out_port(0).data.set_shape(in_node_shape)
def test_select_infer_condition_with_value(self, condition_shape,
else_data_shape,
than_data_shape,
select_output_shape,
condition_value, else_value,
than_value, output_value):
"""
Unit tests generator can sporadic throw exception if we try
to run generator with call numpy array generation functions.
So we need to use lambda function for escape the problem.
"""
condition_value = condition_value(condition_shape)
else_value = else_value(else_data_shape)
than_value = than_value(than_data_shape)
output_value = output_value(select_output_shape)
graph = build_graph_with_attrs(nodes_with_attrs=self.nodes,
edges_with_attrs=self.edges,
update_nodes_attributes=[
('condition_data', {
'shape':
np.array(condition_shape),
'value': condition_value
}),
('else_data', {
'shape':
np.array(else_data_shape),
'value': else_value
}),
('than_data', {
'shape':
np.array(than_data_shape),
'value': than_value
}),
('select_output', {
'shape':
np.array(select_output_shape),
'value':
None
})
])
graph_ref = build_graph_with_attrs(
nodes_with_attrs=self.nodes,
edges_with_attrs=self.edges,
update_nodes_attributes=[('condition_data', {
'shape': np.array(condition_shape),
'value': condition_value
}),
('else_data', {
'shape': np.array(else_data_shape),
'value': else_value
}),
('than_data', {
'shape': np.array(than_data_shape),
'value': than_value
}),
('select_output', {
'shape':
np.array(select_output_shape),
'value': output_value
})])
node = Node(graph, 'select')
Select.infer(node)
if else_value is not None and than_value is not None:
(flag, resp) = compare_graphs(graph,
graph_ref,
'select_output',
check_op_attrs=True)
self.assertTrue(flag, resp)
self.assertTrue(
np.array_equal(graph.nodes['select_output']['value'],
graph_ref.nodes['select_output']['value']))
def replace_pattern(graph: Graph, match: dict):
node = match['op']
if node.name == 'iteration_number_out':
return
# calculate length of context when state of inference becomes meaningful
inputs = []
for n in graph.get_op_nodes(**{'op': 'Parameter'}):
inputs.append(n)
in_nodes = []
for inp in inputs:
for ins in inp.out_port(0).get_destinations():
in_nodes.append(ins.node.name)
context_len = 1
try:
subgraph = invert_sub_graph_between_nodes(
graph, [node.in_port(0).get_source().node.name], in_nodes)
except Error:
return
for n in subgraph:
n_node = Node(graph, n)
if n_node.kind == 'op' and n_node.op == 'Splice':
context_len += len(n_node.context) - 1
if context_len == 1:
return
in_node_port = node.in_port(0).get_source()
in_node_shape = node.in_port(0).data.get_shape()
node.in_port(0).disconnect()
# add Select before saving state to avoid saving garbage
select_node = Select(graph, {
'name': 'select_' + node.name
}).create_node()
zero_else = Const(graph, {
'name': 'zero_else',
'value': np.zeros(in_node_shape)
}).create_node()
select_node.in_port(1).connect(in_node_port)
select_node.in_port(2).connect(zero_else.out_port(0))
# check if we have already appropriate iteration counter
existing_counters = find_pattern_matches(
graph,
nodes=[('mem_in', dict(op='ReadValue')),
('mem_in_data', dict(shape=int64_array([context_len]))),
('crop_mem_in',
dict(op='Crop',
axis=int64_array([1]),
offset=int64_array([1]),
dim=int64_array([context_len - 1]))),
('crop_mem_in_data', dict()),
('concat', dict(op='Concat', axis=1)),
('concat_data', dict()), ('const_1', dict(op='Const')),
('const_1_data', dict()), ('mem_out', dict(op='Assign')),
('crop_out',
dict(op='Crop',
axis=int64_array([1]),
offset=int64_array([0]),
dim=int64_array([1]))), ('crop_out_data', dict()),
('select', dict(op='Select'))],
edges=[('mem_in', 'mem_in_data'), ('mem_in_data', 'crop_mem_in'),
('crop_mem_in', 'crop_mem_in_data'),
('crop_mem_in_data', 'concat', {
'in': 0
}), ('const_1', 'const_1_data'),
('const_1_data', 'concat', {
'in': 1
}), ('concat', 'concat_data'), ('concat_data', 'mem_out'),
('concat_data', 'crop_out'), ('crop_out', 'crop_out_data'),
('crop_out_data', 'select')])
counter_match = next(existing_counters, None)
if counter_match is not None:
ones = Node(graph, inverse_dict(counter_match)['const_1'])
input_port = Node(
graph,
inverse_dict(counter_match)['crop_out']).out_port(0)
else:
init_value_mem_out = create_zero_value_with_batch_from_input(
in_node_port, context_len, np.int32)
mem_out = ReadValue(
graph, {
'name': 'iteration_number',
'variable_id': 'iteration_' + node.name
}).create_node()
mem_out.in_port(0).connect(init_value_mem_out.out_port(0))
cut_first = Crop(
graph, {
'name': 'cut_first',
'axis': int64_array([1]),
'offset': int64_array([1]),
'dim': int64_array([context_len - 1])
}).create_node()
cut_first.in_port(0).connect(mem_out.out_port(0))
ones = Const(graph, {
'name': 'ones',
'value': np.ones([1, 1], dtype=np.int32)
}).create_node()
concat = Concat(graph, {
'name': 'concat_ones',
'in_ports_count': 2,
'axis': 1
}).create_node()
concat.in_port(0).connect(cut_first.out_port(0))
concat.in_port(1).connect(ones.out_port(0))
mem_in = Assign(
graph, {
'name': 'iteration_number_out',
'variable_id': 'iteration_' + node.name
}).create_node()
mem_in.in_port(0).connect(concat.out_port(0))
res = Result(graph, {}).create_node()
mem_in.out_port(0).connect(res.in_port(0))
cut_last = Crop(
graph, {
'name': 'cut_last',
'axis': int64_array([1]),
'offset': int64_array([0]),
'dim': int64_array([1])
}).create_node()
cut_last.in_port(0).connect(concat.out_port(0))
input_port = cut_last.out_port(0)
# Check if data from memory is 1
# if it is True, we have correct data and should proceed with saving it to memory
# else we have not gathered context and have garbage here, shouldn't change initial state of memory
cast_in = Equal(graph, {
'name': input_port.node.name + '/cast_to_bool'
}).create_node()
cast_in.in_port(0).connect(ones.out_port(0))
cast_in.in_port(1).connect(input_port)
select_node.in_port(0).connect(cast_in.out_port(0))
select_node.out_port(0).connect(node.in_port(0))
select_node.out_port(0).data.set_shape(in_node_shape)
def extract(cls, node):
Select.update_node_stat(node, {})
return cls.enabled
def find_and_replace_pattern(self, graph: Graph):
for embedding_segments_mean in graph.get_op_nodes(
op='EmbeddingSegmentsMean'):
embedding_segments_mean_name = embedding_segments_mean.soft_get(
'name', embedding_segments_mean.id)
embedding_table_input = embedding_segments_mean.in_port(0)
segment_ids_input = embedding_segments_mean.in_port(2)
num_segments_input = embedding_segments_mean.in_port(3)
# TODO: support EmbeddingSegmentsMean with specified weights vector.
# now this case has not appeared in models so far so EmbeddingSegmentsOperation fusion
# transformations do not handle it either
if embedding_segments_mean.is_in_port_connected(5):
return
# 1. compute indices membership matrix, i.e. which indices belong to some object
# the shape of this matrix is [num_segments, num_indices]
non_norm_range_1_to_num_segments = create_op_with_const_inputs(
graph, Range, {
0: int64_array(0),
2: int64_array(1)
}, {
'name':
embedding_segments_mean_name + '/Range1ToNumSegments',
'output_type': np.int64
})
num_segments_input.get_connection().add_destination(
non_norm_range_1_to_num_segments.in_port(1))
range_1_to_num_segments = ConvertLike(graph, {
'name':
embedding_segments_mean_name + '/Range1ToNumSegmentsNorm'
}).create_node()
range_1_to_num_segments.in_port(0).connect(
non_norm_range_1_to_num_segments.out_port(0))
num_segments_input.get_connection().add_destination(
range_1_to_num_segments.in_port(1))
unsqueeze_range_1_to_num_segments = create_op_with_const_inputs(
graph, Unsqueeze, {1: int64_array(1)}, {
'name':
embedding_segments_mean_name +
'/Range1ToNumSegmentsUnsqueeze'
})
unsqueeze_range_1_to_num_segments.in_port(0).connect(
range_1_to_num_segments.out_port(0))
unsqueeze_segment_ids = create_op_with_const_inputs(
graph, Unsqueeze, {1: int64_array(0)}, {
'name':
embedding_segments_mean_name + '/SegmentIdsUnsqueeze'
})
segment_ids_input.get_connection().add_destination(
unsqueeze_segment_ids.in_port(0))
boolean_membership_matrix = Equal(graph, {
'name':
embedding_segments_mean_name + '/BooleanMembershipMatrix'
}).create_node()
boolean_membership_matrix.in_port(0).connect(
unsqueeze_range_1_to_num_segments.out_port(0))
boolean_membership_matrix.in_port(1).connect(
unsqueeze_segment_ids.out_port(0))
shape_of_membership_matrix = Shape(graph, {
'name':
embedding_segments_mean_name + '/ShapeOfMembershipMatrix'
}).create_node([boolean_membership_matrix])
one_scalar_constant = Const(
graph, {
'name': embedding_segments_mean_name + '/OneScalar',
'value': int64_array([1])
}).create_node()
one_constant = Broadcast(graph, {
'name':
embedding_segments_mean_name + '/One'
}).create_node([one_scalar_constant, shape_of_membership_matrix])
zero_constant = Const(
graph, {
'name': embedding_segments_mean_name + '/Zero',
'value': int64_array(0)
}).create_node()
membership_matrix = Select(
graph, {
'name': embedding_segments_mean_name + '/MembershipMatrix',
'auto_broadcast': 'numpy'
}).create_node(
[boolean_membership_matrix, one_constant, zero_constant])
# 2. compute a number of indices belong to each object from the batch
# it computes the normalization coefficients
num_indices_per_object = create_op_with_const_inputs(
graph, ReduceSum, {1: int64_array(1)}, {
'name':
embedding_segments_mean_name + '/NumIndicesPerObject'
})
num_indices_per_object.in_port(0).connect(
membership_matrix.out_port(0))
# 3. replace zero coefficient (zero number of indices belong to an object) with one
# because for such object the single default embedding vector is used
where_zero_number = Equal(graph, {
'name':
embedding_segments_mean_name + '/WhereZeroIndicesNumber'
}).create_node([num_indices_per_object, zero_constant])
normalized_num_indices_per_object = Select(
graph, {
'name':
embedding_segments_mean_name + '/NormNumIndicesPerObject',
'auto_broadcast': 'numpy'
}).create_node([
where_zero_number, one_scalar_constant,
num_indices_per_object
])
# 4. cast normalized_num_indices_per_object to the same type as embedding vector table
norm_coefficients = ConvertLike(
graph, {
'name': embedding_segments_mean_name + '/NormCoefficients'
}).create_node()
norm_coefficients.in_port(0).connect(
normalized_num_indices_per_object.out_port(0))
embedding_table_input.get_connection().add_destination(
norm_coefficients.in_port(1))
# 5. replace EmbeddingSegmentMean with EmbeddingSegmentSum
embedding_segments_sum = EmbeddingSegmentsSum(
graph, {
'name':
embedding_segments_mean_name + '/EmbeddingSegmentsSum'
}).create_node()
for in_port in embedding_segments_mean.in_ports():
if embedding_segments_mean.is_in_port_connected(in_port):
embedding_segments_mean.in_port(
in_port).get_connection().set_destination(
embedding_segments_sum.in_port(in_port))
# 6. normalize EmbeddingSegmentSum results by computed coefficients
result_node = Div(graph, {
'name': embedding_segments_mean_name + '/Div'
}).create_node([embedding_segments_sum, norm_coefficients])
embedding_segments_mean.out_port(0).get_connection().set_source(
result_node.out_port(0))
rename_nodes([(embedding_segments_mean,
embedding_segments_mean_name + '/AbandonedName'),
(result_node, embedding_segments_mean_name)])
graph.remove_nodes_from([embedding_segments_mean.id])
def replace_sub_graph(self, graph: Graph, match: Dict[str, Node]):
node = match['op']
name = node.name
min_port_tuple = (node.in_port(1).get_source().node,
node.in_port(1).get_source().idx)
max_port_tuple = (node.in_port(2).get_source().node,
node.in_port(2).get_source().idx)
node.in_port(1).disconnect()
node.in_port(2).disconnect()
# make sure min < max
min_less_max = Less(graph, {
'name': name + '/if_min_less_max'
}).create_node([min_port_tuple, max_port_tuple])
minimum = Select(graph, {
'name': name + '/minimum'
}).create_node([min_less_max, min_port_tuple, max_port_tuple])
maximum = Select(graph, {
'name': name + '/maximum'
}).create_node([min_less_max, max_port_tuple, min_port_tuple])
# to create zero of limits data type, we multiply it by integer zero
zero = create_op_node_with_second_input(graph,
Mul,
int64_array(0),
{'name': name + '/zero'},
input_node=minimum)
# if 0 < min < max: min_adj = 0 and max_adj = max - min
min_greater_zero = Greater(graph, {
'name': name + '/if_minimum_greater_zero'
}).create_node([minimum, zero])
max_minus_min = Sub(graph, {
'name': name + '/max_minus_min'
}).create_node([maximum, minimum])
minimum = Select(graph, {
'name': name + '/first_adj_min'
}).create_node([min_greater_zero, zero, minimum])
maximum = Select(graph, {
'name': name + '/first_adj_max'
}).create_node([min_greater_zero, max_minus_min, maximum])
# if min < max < 0: min_adj = min - max and max_adj = 0
max_less_zero = Less(graph, {
'name': name + '/if_max_less_zero'
}).create_node([maximum, zero])
min_minus_max = Sub(graph, {
'name': name + '/min_minus_max'
}).create_node([minimum, maximum])
minimum = Select(graph, {
'name': name + '/second_adj_min'
}).create_node([max_less_zero, min_minus_max, minimum])
maximum = Select(graph, {
'name': name + '/second_adj_max'
}).create_node([max_less_zero, zero, maximum])
# scale = (max - min) / (2 ^ num_bits - 1),
float_range = Sub(graph, {
'name': name + '/float_range'
}).create_node([maximum, minimum])
quant_min_value, quant_max_value = int(
node.narrow_range), 2**node.num_bits - 1
int_range = Const(
graph,
dict(name=name + '/int_range',
value=quant_max_value - quant_min_value)).create_node()
scale = Div(graph, {
'name': name + '/scale'
}).create_node([float_range, int_range])
# min_adj = scale * round(min / scale)
descaled_min = Div(graph, {
'name': name + '/descaled_min'
}).create_node([minimum, scale])
rounded_descaled_min = Round(graph, {
'name': name + '/rounded_descaled_min'
}).create_node([descaled_min])
min_adj = Mul(graph, {
'name': name + '/min_adj'
}).create_node([scale, rounded_descaled_min])
# max_adj = max + min_adj - min.
adjustment = Sub(graph, {
'name': name + '/limits_adjustment'
}).create_node([min_adj, minimum])
max_adj = Add(graph, {
'name': name + '/max_adj'
}).create_node([maximum, adjustment])
# FakeQuantize operation has 5 inputs instead of 3 inputs in TensorFlow
node.add_input_port(3, skip_if_exist=True)
node.add_input_port(4, skip_if_exist=True)
node.in_port(1).connect(min_adj.out_port(0))
node.in_port(2).connect(max_adj.out_port(0))
node.in_port(3).connect(min_adj.out_port(0))
node.in_port(4).connect(max_adj.out_port(0))
FakeQuantize.update_node_stat(node, {'levels': node['levels']})
def replace_pattern(graph: Graph, match: dict):
node = match['op']
if node.name == 'iteration_number_out':
return
# calculate length of context when state of inference becomes meaningful
inputs = []
for n in graph.get_op_nodes(**{'op': 'Parameter'}):
inputs.append(n)
in_nodes = []
for inp in inputs:
for ins in inp.out_port(0).get_destinations():
in_nodes.append(ins.node.name)
context_len = 1
try:
subgraph = invert_sub_graph_between_nodes(
graph, [node.in_port(0).get_source().node.name], in_nodes)
except Error:
return
for n in subgraph:
n_node = Node(graph, n)
if n_node.kind == 'op' and n_node.op == 'Splice':
context_len += len(n_node.context) - 1
if context_len == 1:
return
in_node_port = node.in_port(0).get_source()
in_node_shape = node.in_port(0).data.get_shape()
node.in_port(0).disconnect()
# add Select before saving state to avoid saving garbage
select_node = Select(graph, {
'name': 'select_' + node.name
}).create_node()
zero_else = Const(graph, {
'name': 'zero_else',
'value': np.zeros(in_node_shape)
}).create_node()
select_node.in_port(1).connect(in_node_port)
select_node.in_port(2).connect(zero_else.out_port(0))
# check if we have already appropriate iteration counter
existing_counters = find_pattern_matches(
graph,
nodes=[('mem_in',
dict(op='Memory',
index=1,
shape=int64_array([context_len]))),
('mem_in_data', dict()),
('crop_mem_in',
dict(op='Crop',
axis=int64_array([1]),
offset=int64_array([1]),
dim=int64_array([context_len - 1]))),
('crop_mem_in_data', dict()),
('concat', dict(op='Concat', axis=1)),
('concat_data', dict()), ('const_1', dict(op='Const')),
('const_1_data', dict()),
('mem_out',
dict(op='Memory',
index=0,
shape=int64_array([context_len]))),
('crop_out',
dict(op='Crop',
axis=int64_array([1]),
offset=int64_array([0]),
dim=int64_array([1]))), ('crop_out_data', dict()),
('select', dict(op='Select'))],
edges=[('mem_in', 'mem_in_data'), ('mem_in_data', 'crop_mem_in'),
('crop_mem_in', 'crop_mem_in_data'),
('crop_mem_in_data', 'concat', {
'in': 0
}), ('const_1', 'const_1_data'),
('const_1_data', 'concat', {
'in': 1
}), ('concat', 'concat_data'), ('concat_data', 'mem_out'),
('concat_data', 'crop_out'), ('crop_out', 'crop_out_data'),
('crop_out_data', 'select')])
counter_match = next(existing_counters, None)
if counter_match is not None:
input_port = Node(
graph,
inverse_dict(counter_match)['crop_out']).out_port(0)
else:
mem_out = Memory(
graph, {
'name': 'iteration_number',
'size': 2,
'index': 1,
'id': 'iteration_' + node.name,
'shape': int64_array([context_len]),
'dst_type': np.int32
}).create_node()
cut_first = Crop(
graph, {
'name': 'cut_first',
'axis': int64_array([1]),
'offset': int64_array([1]),
'dim': int64_array([context_len - 1])
}).create_node()
cut_first.in_port(0).connect(mem_out.out_port(0))
ones = Const(graph, {
'name': 'ones',
'value': np.ones([1, 1], dtype=np.int32)
}).create_node()
concat = Concat(graph, {
'name': 'concat_ones',
'in_ports_count': 2,
'axis': 1
}).create_node()
concat.in_port(0).connect(cut_first.out_port(0))
concat.in_port(1).connect(ones.out_port(0))
mem_in = Memory(
graph, {
'name': 'iteration_number_out',
'size': 2,
'index': 0,
'id': 'iteration_' + node.name,
'shape': int64_array([context_len])
}).create_node()
mem_in.in_port(0).connect(concat.out_port(0))
res = Result(graph, {}).create_node()
mem_in.out_port(0).connect(res.in_port(0))
cut_last = Crop(
graph, {
'name': 'cut_last',
'axis': int64_array([1]),
'offset': int64_array([0]),
'dim': int64_array([1])
}).create_node()
cut_last.in_port(0).connect(concat.out_port(0))
input_port = cut_last.out_port(0)
select_node.in_port(0).connect(input_port)
select_node.out_port(0).connect(node.in_port(0))
select_node.out_port(0).data.set_shape(in_node_shape)
def replace(node: Node, const: Node):
graph = node.graph
shape = const.shape
const_name = const.soft_get('name', const.id)
non_one_dims = np.argwhere(shape != 1).flatten()
one_dims = np.argwhere(shape == 1).flatten()
if not (non_one_dims.size == 1 and 5 < np.prod(shape) < 500):
# (5;500) range is deduced to affect less models
return
value = const.value
if not np.array_equal(
np.arange(0, np.prod(shape), 1).reshape(shape), value):
return
positive_idx = non_one_dims.item(0)
negative_idx = positive_idx - len(shape)
node_name = node.soft_get('name', node.id)
gather = create_op_with_const_inputs(
graph, Gather, {
1: int64_array(negative_idx),
2: int64_array(0)
}, {'name': node_name + '/BroadcastingDim'})
gather_for_const = create_op_with_const_inputs(
graph, Gather, {
1: int64_array(negative_idx),
2: int64_array(0)
}, {'name': const_name + '/BroadcastingDim'})
shapeof_node = Shape(graph, {
'name': const_name + '/ShapeOf'
}).create_node()
shapeof_node.out_port(0).connect(gather_for_const.in_port(0))
equal_node = create_op_with_const_inputs(
graph, Equal, {1: int64_array(1)},
{'name': node_name + '/ConstOne'})
gather.out_port(0).connect(equal_node.in_port(0))
select_node = Select(graph, {
'name': node_name + '/Select',
'auto_broadcast': 'numpy'
}).create_node([equal_node, gather_for_const, gather])
const.out_port(0).connect(shapeof_node.in_port(0))
range_node = create_op_with_const_inputs(
graph, Range, {
0: np.array(0, dtype=value.dtype),
2: np.array(1, dtype=value.dtype)
}, {
'name': const_name + '/Range',
'dtype': value.dtype
})
select_node.out_port(0).connect(range_node.in_port(1))
node.in_port(1).get_connection().add_destination(gather.in_port(0))
node.in_port(0).get_connection().set_source(range_node.out_port(0))
if one_dims.size:
unsqueeze = create_op_node_with_second_input(
graph, Unsqueeze, one_dims,
{'name': const_name + '/KeepShape'})
range_node.out_port(0).get_connection().insert_node(unsqueeze)
rename_nodes([(const, const_name + '/ToBeDeleted'),
(unsqueeze, const_name)])
else:
rename_nodes([(const, const_name + '/ToBeDeleted'),
(range_node, const_name)])
def extract(cls, node: Node):
Select.update_node_stat(node, {
'format': 'tf',
})
return cls.enabled
def extract(node):
Select.update_node_stat(node, {})
return __class__.enabled
def replace_sub_graph(self, graph: Graph, match: dict):
seq_len_tf = match['seq_len']
transpose_tf = match['transpose']
ctc_greedy_decoder_tf = match['ctc_greedy_decoder']
cast_tf = match['cast']
ctc_loss_tf = match['ctc_loss']
sparse_to_dense_tf = match['sparse_to_dense']
output_sparse_to_dense_name = sparse_to_dense_tf.soft_get(
'name', sparse_to_dense_tf.id)
output_ctc_loss_name = ctc_loss_tf.soft_get('name', ctc_loss_tf.id)
ctc_greedy_decoder_tf_name = ctc_greedy_decoder_tf.soft_get(
'name', ctc_greedy_decoder_tf.id)
log.debug(
'Found CTCLossFrontReplacer pattern after {} with name {}'.format(
ctc_greedy_decoder_tf.op, ctc_greedy_decoder_tf.name))
# create sequence mask node, sub-graph for transforming into sequence length and connect with consumers
seq_len_tf_shape = seq_len_tf.soft_get('shape', None)
if seq_len_tf_shape is None or len(seq_len_tf_shape) != 2:
raise Error(
'The sequence length that is the second input to the CTCGreedyDecoder node "{}"'
' must be specified in a mask format.'.format(
ctc_greedy_decoder_tf_name))
log.error(
'The format of input sequence length has been changed to a mask format',
extra={'is_warning': True})
seq_len_tf_type = seq_len_tf.soft_get('data_type', None)
seq_len_tf_name = seq_len_tf.soft_get('name', seq_len_tf.id)
seq_mask_placeholder = Parameter(
graph, {
'name': seq_len_tf_name,
'shape': seq_len_tf_shape,
'data_type': seq_len_tf_type
}).create_node()
reduce_to_seq_len_node = create_op_with_const_inputs(
graph, ReduceSum, {1: np.array(1, dtype=np.int32)}, {
'name': seq_len_tf_name + '/ReduceToSeqLen',
'keep_dims': False
})
reduce_to_seq_len_node.in_port(0).connect(
seq_mask_placeholder.out_port(0))
seq_len_tf.out_port(0).get_connection().set_source(
reduce_to_seq_len_node.out_port(0))
cast_fp_type = data_type_str_to_np(graph.graph['cmd_params'].data_type)
casted_seq_mask_node = Cast(graph, {
'name': seq_len_tf_name + '/CastToFP32',
'dst_type': cast_fp_type
}).create_node()
casted_seq_mask_node.in_port(0).connect(
seq_mask_placeholder.out_port(0))
permuted_casted_seq_mask = create_op_with_const_inputs(
graph, Transpose, {1: int64_array([1, 0])},
{'name': seq_len_tf_name + '/Permute'})
permuted_casted_seq_mask.in_port(0).connect(
casted_seq_mask_node.out_port(0))
rename_nodes([(seq_len_tf, seq_len_tf_name + '/AbandonedName'),
(seq_mask_placeholder, seq_len_tf_name)])
# create CTCGreedyDecoder node and set mask node
ctc_merge_repeated_i = ctc_greedy_decoder_tf.soft_get(
'ctc_merge_repeated', ctc_greedy_decoder_tf.id)
ctc_greedy_decoder = CTCGreedyDecoderOp(
graph, {
'name': output_sparse_to_dense_name,
'ctc_merge_repeated': ctc_merge_repeated_i
}).create_node()
ctc_greedy_decoder.in_port(1).connect(
permuted_casted_seq_mask.out_port(0))
rename_nodes([(sparse_to_dense_tf,
output_sparse_to_dense_name + '/AbandonedName'),
(ctc_greedy_decoder, output_sparse_to_dense_name)])
# create CTCLoss node and set attributes
assert ctc_loss_tf.has_valid('preprocess_collapse_repeated'), \
'The CTCLoss node "{}" misses "preprocess_collapse_repeated" attribute'.format(output_ctc_loss_name)
assert ctc_loss_tf.has_valid('ctc_merge_repeated'), \
'The CTCLoss node "{}" misses "ctc_merge_repeated" attribute'.format(output_ctc_loss_name)
assert ctc_loss_tf.has_valid('unique'), \
'The CTCLoss node "{}" misses "unique" attribute'.format(output_ctc_loss_name)
preprocess_collapse_repeated = ctc_loss_tf.preprocess_collapse_repeated
ctc_merge_repeated = ctc_loss_tf.ctc_merge_repeated
unique = ctc_loss_tf.unique
ctc_loss = CTCLoss(
graph, {
'name': output_ctc_loss_name,
'preprocess_collapse_repeated': preprocess_collapse_repeated,
'ctc_merge_repeated': ctc_merge_repeated,
'unique': unique
}).create_node()
rename_nodes([(ctc_loss_tf, output_ctc_loss_name + '/AbandonedName'),
(ctc_loss, output_ctc_loss_name)])
# connect logits
ctc_greedy_decoder_tf.in_port(0).get_connection().set_destination(
ctc_greedy_decoder.in_port(0))
ctc_loss.in_port(0).disconnect()
transpose_tf.in_port(0).get_connection().add_destination(
ctc_loss.in_port(0))
# connect logit lengths
ctc_greedy_decoder_tf.in_port(1).disconnect()
ctc_loss.in_port(1).connect(reduce_to_seq_len_node.out_port(0))
# connect labels to ctc_loss
squeeze_op = create_op_with_const_inputs(graph, Squeeze,
{1: int64_array([2, 3])})
cast_labels_op = Cast(
graph, {
'name': output_sparse_to_dense_name + '/CastLabels',
'dst_type': np.int32
}).create_node()
squeeze_op.in_port(0).connect(ctc_greedy_decoder.out_port(0))
cast_labels_op.in_port(0).connect(squeeze_op.out_port(0))
ctc_loss.in_port(2).connect(cast_labels_op.out_port(0))
# connect label lengths
equal_op = create_op_with_const_inputs(
graph, Equal, {1: np.array([-1], dtype=np.int32)},
{'name': output_sparse_to_dense_name + '/Equal'})
equal_op.in_port(0).connect(cast_labels_op.out_port(0))
labels_shape_op = Shape(
graph, {
'name': output_sparse_to_dense_name + '/ShapeOf'
}).create_node()
labels_shape_op.in_port(0).connect(equal_op.out_port(0))
broadcast_one = create_op_with_const_inputs(
graph, Broadcast, {0: np.array([1], dtype=np.int32)}, {
'mode': 'numpy',
'name': output_sparse_to_dense_name + '/One'
})
broadcast_one.in_port(1).connect(labels_shape_op.out_port(0))
broadcast_zero = create_op_with_const_inputs(
graph, Broadcast, {0: np.array([0], dtype=np.int32)}, {
'mode': 'numpy',
'name': output_sparse_to_dense_name + '/Zero'
})
broadcast_zero.in_port(1).connect(labels_shape_op.out_port(0))
select_node = Select(graph, {
'name': output_sparse_to_dense_name + '/Select'
}).create_node()
select_node.in_port(0).connect(equal_op.out_port(0))
select_node.in_port(1).connect(broadcast_zero.out_port(0))
select_node.in_port(2).connect(broadcast_one.out_port(0))
label_length_node = create_op_with_const_inputs(
graph,
ReduceSum, {1: int64_array([1])},
op_attrs={
'name': output_sparse_to_dense_name + '/LabelLength',
'keep_dims': False
})
label_length_node.in_port(0).connect(select_node.out_port(0))
ctc_loss.in_port(3).connect(label_length_node.out_port(0))
# set source for output of new sub-graph and remove old nodes
ctc_loss_tf.out_port(0).get_connection().set_source(
ctc_loss.out_port(0))
graph.remove_nodes_from([
ctc_greedy_decoder_tf.id, ctc_loss_tf.id, cast_tf.id,
sparse_to_dense_tf.id
])
def extract(node: Node):
Select.update_node_stat(node, {
'format': 'tf',
})
return __class__.enabled
def dequantize_data(fake_quantize: Node, dst_type: type,
quantized_type: type) -> Node:
graph = fake_quantize.graph
quantized_data = fake_quantize.in_port(0).get_source().node
name = fake_quantize.soft_get('name', fake_quantize.id)
assert quantized_data.soft_get('type') == 'Convert' and quantized_data.dst_type == quantized_type, \
'Weights aren`t compressed as expected for node {}'.format(fake_quantize.soft_get('name', fake_quantize.id))
dequantizing_cast = Cast(
graph,
dict(name=quantized_data.name +
"/to_{}".format(np_data_type_to_destination_type(dst_type)),
dst_type=dst_type,
stop_value_propagation=True)).create_node()
fake_quantize.in_port(0).get_connection().set_destination(
dequantizing_cast.in_port(0))
# limits of dequantize
in_low = fake_quantize.in_port(1).get_source()
in_high = fake_quantize.in_port(2).get_source()
out_low = fake_quantize.in_port(3).get_source()
out_high = fake_quantize.in_port(4).get_source()
# scale calculation
output_range = Sub(graph, {
'name': name + '/output_range'
}).create_node()
output_range.in_port(0).connect(out_high)
output_range.in_port(1).connect(out_low)
input_range = Sub(graph, {'name': name + '/input_range'}).create_node()
input_range.in_port(0).connect(in_high)
input_range.in_port(1).connect(in_low)
scale = Div(graph, {'name': name + '/scale'}).create_node()
scale.in_port(0).connect(output_range.out_port(0))
scale.in_port(1).connect(input_range.out_port(0))
# shift calculation
descaled_output_low = Div(graph, {
'name': name + '/descaled_output_low'
}).create_node()
descaled_output_low.in_port(0).connect(out_low)
descaled_output_low.in_port(1).connect(scale.out_port(0))
shift = Sub(graph, {'name': name + '/shift'}).create_node()
shift.in_port(0).connect(in_low)
shift.in_port(1).connect(descaled_output_low.out_port(0))
zero = Const(graph, {
'name': name + '/zero',
'value': np.array(0, dtype=dst_type)
}).create_node()
scale_eq_zero = Equal(graph, {
'name': name + '/scale_eq_zero'
}).create_node()
scale_eq_zero.in_port(0).connect(scale.out_port(0))
scale_eq_zero.in_port(1).connect(zero.out_port(0))
zero_point = Select(graph, {
'name': name + '/zero_point'
}).create_node()
zero_point.in_port(0).connect(scale_eq_zero.out_port(0))
zero_point.in_port(1).connect(zero.out_port(0))
zero_point.in_port(2).connect(shift.out_port(0))
# DeQuantize(x) == Mul(Sub(x, zero_point), scale)
sub_zp = Sub(graph, {'name': name + '/minus_zp'}).create_node()
sub_zp.in_port(0).connect(dequantizing_cast.out_port(0))
sub_zp.in_port(1).connect(zero_point.out_port(0))
mul_scale = Mul(graph, {
'name': name + '/mulpiply_by_scale'
}).create_node()
mul_scale.in_port(0).connect(sub_zp.out_port(0))
mul_scale.in_port(1).connect(scale.out_port(0))
fake_quantize.out_port(0).get_connection().set_source(
mul_scale.out_port(0))
graph.remove_nodes_from([fake_quantize.id, fake_quantize.out_node(0)])
