def _fused_batch_norm_decomposition(graph: Graph, tinput: Node, toutput: Node, gamma: Node, beta: Node,
mean: np.ndarray, variance: np.ndarray, can_be_fused=True):
"""
This is common function for TF, Caffe and MXNet
It creates Mul->Add->Mul->Add subgraph
"""
shape = tinput.shape
# Create first Mul & Add operations
mul1_node = Mul(graph, dict(name="Mul1_", can_be_fused=can_be_fused))
add1_node = Add(graph, dict(name="Add1_", can_be_fused=can_be_fused))
mul1_data = Op.create_input_data_node(graph, "data_mul_", np.array(mean))
add1_data = Op.create_input_data_node(graph, "data_add_", np.array(variance))
# Broadcast const from scalar
# We can broadcast only when const.value is scalar
if gamma.shape[0] != gamma.value.shape[0]:
gamma.value.resize(gamma.shape)
gamma.value.fill(gamma.value[0])
# Create second Mul & Add
mul2_node = Mul(graph, dict(name="Mul2_", can_be_fused=can_be_fused))
add2_node = Add(graph, dict(name="Add2_", can_be_fused=can_be_fused))
add2_node.create_node_with_data(
inputs=[mul2_node.create_node_with_data(
inputs=[add1_node.create_node_with_data(
inputs=[mul1_node.create_node_with_data(inputs=[tinput, mul1_data]),
add1_data]),
gamma]),
beta],
data_nodes=toutput)
def replace_pattern(self, graph: nx.MultiDiGraph, match: dict):
"""
Adds Normalize layer weights, which are required by Inference Engine,
but do not always exist in MXNet model.
L2Normalization is mapped to Normalize layer
so we need to generate Normalize weights filled with ones.
Parameters
----------
graph : nx.MultiDiGraph
Graph with loaded model.
match : dict
Patterns which were found in graph structure.
"""
l2_normalization_node = match['l2_normalization']
if len(l2_normalization_node.in_nodes()) < 2:
value = np.full([l2_normalization_node.in_node(0).shape[1]],
1.0,
dtype=np.float32)
weights_node = Op.create_input_data_node(
graph,
name=l2_normalization_node['name'] + '_weights',
value=value)
create_edge(weights_node,
l2_normalization_node,
out_port=0,
in_port=1,
edge_attrs={'bin': 'weights'})
def apply_mean_value(graph: Graph, input_node: Node,
node_mean_scale_values: dict):
if 'mean' in node_mean_scale_values and node_mean_scale_values[
'mean'] is not None:
if all([x == 0 for x in node_mean_scale_values['mean']]):
return
out_node = input_node.out_node()
if not input_node.has_valid('shape'):
raise Error("Node {} has not valid shape attribute".format(
input_node.id))
input_shape = input_node.shape
# Create Add node
graph.remove_edge(input_node.id, out_node.id)
value = np.array(node_mean_scale_values['mean']) * (-1)
add_node = Add(graph, dict(name="Add_"))
add_data = Op.create_input_data_node(graph, "data_add_",
np.array(value))
Op.expand_node_shape(add_data,
(len(input_shape) -
2 if graph.graph['layout'] == 'NCHW' else 0))
add_input = Op.create_data_node(graph, input_node,
{'shape': out_node.shape})
add_node.create_node_with_data(inputs=[add_input, add_data],
data_nodes=out_node)
def apply_scale(graph: Graph, input_node: Node,
node_mean_scale_values: dict):
if 'scale' in node_mean_scale_values and node_mean_scale_values[
'scale'] is not None:
if all([x == 1 for x in node_mean_scale_values['scale']]):
return
out_node = input_node.out_node()
if not input_node.has_valid('shape'):
raise Error("Node {} has not valid shape attribute".format(
input_node.id))
input_shape = input_node.shape
# Create Mul node
value = 1 / np.array(node_mean_scale_values['scale'])
graph.remove_edge(input_node.id, out_node.id)
mul_node = Mul(graph, dict(name="Mul_"))
mul_data = Op.create_input_data_node(graph, "data_mul_",
np.array(value))
Op.expand_node_shape(mul_data,
(len(input_shape) -
2 if graph.graph['layout'] == 'NCHW' else 0))
mul_input = Op.create_data_node(graph, input_node,
{'shape': out_node.shape})
mul_node.create_node_with_data(inputs=[mul_input, mul_data],
data_nodes=out_node)
def duplicate_shared_weights(graph: nx.MultiDiGraph):
"""
This function finds all const data nodes that have more that one consumer and then duplicate them
"""
data_nodes = [
Node(graph, id) for id in graph.nodes()
if Node(graph, id).soft_get('kind') == 'data'
]
for node in data_nodes:
# Check that node has const values and more than one consumer
if len(node.out_nodes()) > 1 and node.value is not None:
# Here we delete all edges between base node and it's consumers (except first), and then duplicate this
# node to connect with other consumers
while len(node.out_nodes()) > 1:
out_node = node.out_node(1)
if len(graph.get_edge_data(node.id, out_node.id)) != 1:
raise Error(
'There is more than one edge from {} node to {} node.'.
format(node.id, out_node.id))
e_attrs = graph.get_edge_data(node.id, out_node.id)[0]
graph.remove_edge(node.id, out_node.id)
data = Op.create_input_data_node(graph,
"Copy_{}".format(node.id),
np.array(node.value),
graph.node[node.id])
graph.add_edges_from([(data.id, out_node.id, e_attrs)])
def _scale_input_action_mul(graph: nx.MultiDiGraph, match: dict, scale: float):
assert (len(match['placeholder'].out_nodes()))
tinput = match['placeholder']
if not tinput.has_valid('shape'):
raise Error("Node {} has not valid shape attribute".format(tinput.id))
input_shape = tinput.shape
toutput = match['data']
# Create Mul node
value = np.array([1 / scale])
# Disconnect input with data node
graph.remove_edge(tinput.id, toutput.id)
# Create Mul node
mul_node = Mul(graph, dict(name="Mul1_"))
mul_data = Op.create_input_data_node(graph, "data_mul_scale_",
np.array(value))
Op.expand_node_shape(
mul_data,
len(input_shape) - 2 if graph.graph['layout'] == 'NCHW' else 0)
mul_input = Op.create_data_node(graph, tinput, {'shape': toutput.shape})
mul_node.create_node_with_data(inputs=[mul_input, mul_data],
data_nodes=toutput)
def find_and_replace_pattern(self, graph: nx.MultiDiGraph):
data_nodes = [
Node(graph, node) for node in graph.node
if Node(graph, node).kind == 'data'
]
for node in data_nodes:
# Get all requested shapes for current node
# This mapping will contain pairs like {shape:[list of consumers nodes]}
mapping = {}
for consumer in node.out_nodes():
edge_attrs = graph.get_edge_data(node.id, consumer.id)[0]
if 'new_shape' in edge_attrs:
if np.array_equal(edge_attrs['new_shape'], node.shape):
continue
new_shape = tuple([x for x in edge_attrs['new_shape']])
if not new_shape in mapping:
mapping.update({new_shape: [consumer]})
else:
mapping[new_shape].append(consumer)
if node.has_valid('value'):
# Check that requested shape are the same
# In case if they are different, we duplicate them
for shape_key in mapping.keys():
shape = list(shape_key)
new_value = np.reshape(node.value, shape)
node_copy = Op.create_input_data_node(
graph, node.id + '/copy', value=np.array(new_value))
for consumer in mapping[shape_key]:
edge_attrs = graph.get_edge_data(node.id,
consumer.id)[0]
del edge_attrs['new_shape']
# Remove edge from previous data node and connect new data node with its consumer
graph.remove_edge(node.id, consumer.id)
graph.add_edge(node_copy.id, consumer.id, **edge_attrs)
else:
# Insert Reshape layer between data node and consumer
for shape_key in mapping.keys():
shape = list(shape_key)
reshape = Reshape(graph,
attrs={
'dim': shape,
'name': 'EltwiseReshapeNormalization'
})
reshape_data = reshape.create_node_with_data(inputs=[node])
# Iterate over consumers and reconnect them to Reshape layer output
for consumer in mapping[shape_key]:
edge_attrs = graph.get_edge_data(node.id,
consumer.id)[0]
del edge_attrs['new_shape']
# Reconnect edge from original data node to Reshape output datanode
graph.remove_edge(node.id, consumer.id)
graph.add_edge(reshape_data.id, consumer.id,
**edge_attrs)
def find_and_replace_pattern(self, graph: Graph):
"""
This function finds all const data nodes that have more that one consumer and then duplicate them
"""
data_nodes = [Node(graph, id) for id in graph.nodes() if Node(graph, id).soft_get('kind') == 'data']
for node in data_nodes:
# Check that node has const values and more than one consumer
if len(node.in_nodes()) and node.in_node().soft_get('type') == 'Const' and len(node.out_nodes()) > 1 and \
node.value is not None:
# Here we delete all edges between base node and it's consumers (except first), and then duplicate this
# node to connect with other consumers
for v, d in node.get_outputs():
out_node = Node(graph, v)
e_attrs = d
graph.remove_edge(node.id, out_node.id)
data = Op.create_input_data_node(graph, "Copy_{}".format(node.id), np.array(node.value),
graph.node[node.id])
graph.add_edges_from([(data.id, out_node.id, e_attrs)])
def _fuse_linear_sequence(graph: nx.MultiDiGraph, start_node: Node):
"""
This function finds the sequence of Mul/Add operations and replaces this sequence with two ops (Mul->Add).
:param graph:
:param start_node: The first operation of the sequence
"""
fnodes = [start_node]
while True:
node = fnodes[-1]
data_node = node.out_node()
if (len(data_node.out_nodes()) != 1):
break
if (data_node.out_node().op in ['Mul', 'Add']) and get_value_id(
data_node.out_node()) is not None and data_node.out_node(
).soft_get('can_be_fused') == True:
fnodes.append(data_node.out_node())
else:
break
if len(fnodes) == 1 or (len(fnodes) == 2 and fnodes[0].op == 'Mul'
and fnodes[1].op == 'Add'):
return False
input_shape = start_node.in_node(get_tensor_id(start_node)).shape
init_dims_cnt = len(
input_shape) - 2 if graph.graph['layout'] == 'NCHW' else 1
mul = np.ones([1 for x in range(init_dims_cnt)])
add = np.zeros([1 for x in range(init_dims_cnt)])
first_mul_name = None
first_add_name = None
for idx in range(len(fnodes)):
node = fnodes[idx]
const_node = get_value_id(node)
if node.op == 'Mul':
if first_mul_name is None:
first_mul_name = node.name
mul = mul * node.in_node(const_node).value
add = add * node.in_node(const_node).value
elif node.op == 'Add':
if first_add_name is None:
first_add_name = node.name
add = add + node.in_node(const_node).value
# If mul is scalar we broadcast it to biases shape
if mul.shape != add.shape and len(mul.shape) == 1 and mul.shape[0] == 1:
mul = np.array([mul[0] for x in range(add.shape[0])])
assert (np.array_equal(fnodes[0].in_node(get_tensor_id(fnodes[0])).shape,
fnodes[-1].out_node().shape))
mul_node = Mul(
graph,
dict(name=first_mul_name +
'/Fused_Mul_' if first_mul_name is not None else ''))
add_node = Add(
graph,
dict(name=first_add_name +
'/Fused_Add_' if first_add_name is not None else ''))
in_node = fnodes[0].in_node(get_tensor_id(fnodes[0]))
out_node = fnodes[-1].out_node()
graph.remove_edge(in_node.id, fnodes[0].id)
graph.remove_edge(fnodes[-1].id, out_node.id)
# Remove deleted subgraph
for node in fnodes:
for tmp_node in node.in_nodes().values():
# Remove node only if it has one consumer (for case with shared weights)
if len(tmp_node.out_nodes()) == 1:
graph.remove_node(tmp_node.id)
for tmp_node in node.out_nodes().values():
graph.remove_node(tmp_node.id)
graph.remove_node(node.id)
"""
Four cases considered below:
1. Mul and Add have valid values (mul value != 1 and add value != 0)
2. Only Mul has valid values, so we add only Mul node
3. Only Add has valid values, so we add only Add node
4. When Mul and Add has not valid values we just merge two data nodes
"""
if any([x != 0
for x in np.nditer(add)]) and any([x != 1
for x in np.nditer(mul)]):
data_mul = Op.create_input_data_node(graph, "data_mul_", np.array(mul))
data_add = Op.create_input_data_node(graph, "data_add_", np.array(add))
add_node.create_node_with_data(inputs=[
mul_node.create_node_with_data([in_node, data_mul]), data_add
],
data_nodes=out_node)
elif any([x != 1 for x in np.nditer(mul)]):
data_mul = Op.create_input_data_node(graph, "data_mul_", np.array(mul))
mul_node.create_node_with_data(inputs=[in_node, data_mul],
data_nodes=out_node)
elif any([x != 0 for x in np.nditer(add)]):
data_add = Op.create_input_data_node(graph, "data_add_", np.array(add))
add_node.create_node_with_data(inputs=[in_node, data_add],
data_nodes=out_node)
else:
merge_data_nodes(graph, out_node, in_node)
graph.remove_node(in_node.id)
log.debug('Fused {} operations'.format(len(fnodes)))
return True