def convert_add_to_scaleshift(graph: nx.MultiDiGraph):
for n in list(graph.nodes()):
node = Node(graph, n)
if node.has('op') and (node.op == 'BiasAdd' or node.op
== 'Add') and len(node.in_nodes()) == 2:
tensor_id, value_id = get_tensor_id(node), get_value_id(node)
if tensor_id is not None and value_id is not None and node.soft_get(
'can_be_scaleshift') is not False:
node['type'] = 'ScaleShift'
node['op'] = 'ScaleShift'
node.in_node(value_id).value = np.squeeze(
node.in_node(value_id).value)
node.in_node(value_id).shape = node.in_node(
value_id).value.shape
# if the node was created with eltwise then it has attribute 'operation' which should be removed from
# the IR
if node.has('operation'):
del graph.node[n]['operation']
bias_data = node.in_node(value_id)
graph[bias_data.node][node.node][0]['in'] = 2
graph[bias_data.node][node.node][0]['bin'] = 'biases'
input_data = node.in_node(tensor_id)
graph[input_data.node][node.node][0]['in'] = 0
update_ie_fields(graph.node[node.id])
weights_id = unique_id(graph, 'weights_')
graph.add_node(
weights_id,
**add_attrs_props(
dict(kind='data',
precision="FP32",
name=weights_id,
value=None,
shape=None,
data_type=None,
infer=None)))
wnode = Node(graph, weights_id)
wnode['value'] = np.full_like(bias_data.value,
1,
dtype=np.float32)
wnode['shape'] = np.array(wnode['value'].shape)
graph.add_edges_from([
(weights_id, node.node, {
'in': 1,
'bin': 'weights'
}),
])
def muladd_to_scaleshift_action(graph: Graph, match: dict):
mul = match['mul']
add = match['add']
output = match['output']
# Pass works correctly only in case when node have only 1 output
if len(mul.out_port(0).get_destinations()) > 1:
return
if mul.soft_get('can_be_scaleshift') is False or add.soft_get(
'can_be_scaleshift') is False:
return
mul_weights_id = get_value_id(mul)
mul_input_id = get_tensor_id(mul)
add_weights_id = get_value_id(add)
if mul_weights_id is None:
log.debug("Mul->Add to ScaleShift: Mul {} has no weights".format(
mul.name))
return
if mul_input_id is None:
log.debug("Mul->Add to ScaleShift: Mul {} has no input".format(
mul.name))
return
if add_weights_id is None:
log.debug("Mul->Add to ScaleShift: Add {} has no weights".format(
add.name))
return
input = mul.in_node(mul_input_id)
weights = mul.in_node(mul_weights_id)
bias = add.in_node(add_weights_id)
# Transform values
weights.value = np.squeeze(weights.value)
weights.shape = np.array(weights.value.shape, dtype=np.int64)
bias.value = np.squeeze(bias.value)
bias.shape = np.array(bias.value.shape, dtype=np.int64)
# Broadcast weights if they are scalar
if weights.value.ndim == 0 and bias.value.ndim == 1:
weights.value = np.full(bias.shape, weights.value.item())
weights.shape = np.array(weights.value.shape, dtype=np.int64)
if bias.shape != weights.shape:
log.warning('Mul->Add to ScaleShift conversion stoped {} != {}'.format(
weights.shape, bias.shape))
return
if bias.value.ndim != weights.value.ndim or bias.value.size != weights.value.size:
log.debug(
"Skipping Mul->Add to ScaleShift conversion for nodes {}, {} because of different weights "
"and biases".format(mul.name, add.name))
return
if bias.value.size == 1 and weights.value.size == 1:
log.debug(
"Skipping Mul->Add to ScaleShift conversion for nodes {}, {}. Will be converted to Power"
"".format(mul.name, add.name))
return
op_name = "ScaleShift"
log.debug(
"Fusing Mul->Add to {}. Input nodes: {} and {}, bias.shape = {}, weights.shape = {}"
"".format(op_name, mul.id, add.id, bias.shape, weights.shape))
graph.remove_edge(input.node, mul.id)
graph.remove_edge(weights.node, mul.id)
graph.remove_edge(bias.node, add.id)
graph.remove_edge(add.node, output.id)
op_node = graph.unique_id(mul.name + '/Fused{}_'.format(op_name))
graph.add_node(
op_node,
**add_attrs_props(
dict(kind='op',
type=op_name,
name=op_node,
op=op_name,
data_type=input.data_type)))
scsh = Node(graph, op_node)
scsh.add_input_port(0)
scsh.add_input_port(1)
scsh.add_input_port(2)
scsh.add_output_port(0)
update_ie_fields(graph.node[op_node])
graph.add_edges_from([(input.node, op_node, {
'in': 0
}), (weights.node, op_node, {
'in': 1,
'bin': 'weights'
}), (bias.node, op_node, {
'in': 2,
'bin': 'biases'
}), (op_node, output.node, {
'out': 0
})])
return
def _fuse_linear_sequence(graph: nx.MultiDiGraph, start_node: Node):
"""
This function finds the sequence of Mul/Add operations and replaces this sequence with two ops (Mul->Add).
:param graph:
:param start_node: The first operation of the sequence
"""
fnodes = [start_node]
while True:
node = fnodes[-1]
data_node = node.out_node()
if (len(data_node.out_nodes()) != 1):
break
if (data_node.out_node().op in ['Mul', 'Add']) and get_value_id(
data_node.out_node()) is not None and data_node.out_node(
).soft_get('can_be_fused') == True:
fnodes.append(data_node.out_node())
else:
break
if len(fnodes) == 1 or (len(fnodes) == 2 and fnodes[0].op == 'Mul'
and fnodes[1].op == 'Add'):
return False
input_shape = start_node.in_node(get_tensor_id(start_node)).shape
init_dims_cnt = len(
input_shape) - 2 if graph.graph['layout'] == 'NCHW' else 1
mul = np.ones([1 for x in range(init_dims_cnt)])
add = np.zeros([1 for x in range(init_dims_cnt)])
first_mul_name = None
first_add_name = None
for idx in range(len(fnodes)):
node = fnodes[idx]
const_node = get_value_id(node)
if node.op == 'Mul':
if first_mul_name is None:
first_mul_name = node.name
mul = mul * node.in_node(const_node).value
add = add * node.in_node(const_node).value
elif node.op == 'Add':
if first_add_name is None:
first_add_name = node.name
add = add + node.in_node(const_node).value
# If mul is scalar we broadcast it to biases shape
if mul.shape != add.shape and len(mul.shape) == 1 and mul.shape[0] == 1:
mul = np.array([mul[0] for x in range(add.shape[0])])
assert (np.array_equal(fnodes[0].in_node(get_tensor_id(fnodes[0])).shape,
fnodes[-1].out_node().shape))
mul_node = Mul(
graph,
dict(name=first_mul_name +
'/Fused_Mul_' if first_mul_name is not None else ''))
add_node = Add(
graph,
dict(name=first_add_name +
'/Fused_Add_' if first_add_name is not None else ''))
in_node = fnodes[0].in_node(get_tensor_id(fnodes[0]))
out_node = fnodes[-1].out_node()
graph.remove_edge(in_node.id, fnodes[0].id)
graph.remove_edge(fnodes[-1].id, out_node.id)
# Remove deleted subgraph
for node in fnodes:
for tmp_node in node.in_nodes().values():
# Remove node only if it has one consumer (for case with shared weights)
if len(tmp_node.out_nodes()) == 1:
graph.remove_node(tmp_node.id)
for tmp_node in node.out_nodes().values():
graph.remove_node(tmp_node.id)
graph.remove_node(node.id)
"""
Four cases considered below:
1. Mul and Add have valid values (mul value != 1 and add value != 0)
2. Only Mul has valid values, so we add only Mul node
3. Only Add has valid values, so we add only Add node
4. When Mul and Add has not valid values we just merge two data nodes
"""
if any([x != 0
for x in np.nditer(add)]) and any([x != 1
for x in np.nditer(mul)]):
data_mul = Op.create_input_data_node(graph, "data_mul_", np.array(mul))
data_add = Op.create_input_data_node(graph, "data_add_", np.array(add))
add_node.create_node_with_data(inputs=[
mul_node.create_node_with_data([in_node, data_mul]), data_add
],
data_nodes=out_node)
elif any([x != 1 for x in np.nditer(mul)]):
data_mul = Op.create_input_data_node(graph, "data_mul_", np.array(mul))
mul_node.create_node_with_data(inputs=[in_node, data_mul],
data_nodes=out_node)
elif any([x != 0 for x in np.nditer(add)]):
data_add = Op.create_input_data_node(graph, "data_add_", np.array(add))
add_node.create_node_with_data(inputs=[in_node, data_add],
data_nodes=out_node)
else:
merge_data_nodes(graph, out_node, in_node)
graph.remove_node(in_node.id)
log.debug('Fused {} operations'.format(len(fnodes)))
return True
def _fuse_mul(graph: nx.MultiDiGraph,
node: Node,
fuse_nodes: list,
backward: bool = True):
"""
This function takes Mul node and array of convolution/fc nodes for further fusion
Parameters
----------
x : bool
If backward is False, that means that Convolution/FC goes after Mul node
else means that Mul goes after Convolutions/FC
:param backward:
:param fuse_nodes:
:param node:
:param graph:
"""
is_fused = False
const_id, tensor_id = get_value_id(node), get_tensor_id(node)
if const_id is None or tensor_id is None:
log.warning(
'Cannot do fuse_mul for node {} because this node has wrong inputs'
.format(node.id))
return False
for fuse_node in fuse_nodes:
if fuse_node.soft_get('can_be_fused') == False:
log.warning(
'Node {} can\'t be used in fusing due to user specified attr can_be_fused = False'
.format(fuse_node.id))
return False
if len(fuse_node.in_nodes()) < 2:
log.warning('Node {} has no weights node'.format(fuse_node.id))
return False
if not fuse_node.has_valid('layout'):
log.warning('Node {} has no layout attr'.format(fuse_node.id))
return False
weights_node = fuse_node.in_node(1)
if not weights_node.has_valid(
'output_channel_dim') or not weights_node.has_valid(
'input_channel_dim'):
log.warning(
'Cannot do fuse_mul for node {} because there is no field ' +
'output_channel_dim and/or input_channel_dim in weights.'.
format(fuse_node.soft_get('name')))
return False
inp_ch, out_ch = weights_node.input_channel_dim, weights_node.output_channel_dim
if max(inp_ch, out_ch) >= len(weights_node.shape):
log.warning('Node {} has wrong weights shape'.format(fuse_node.id))
return False
for fuse_node in fuse_nodes:
weights_node = fuse_node.in_node(1)
value = np.array(node.in_node(const_id).value)
value = np.squeeze(value)
# TODO : ch_dim should be equal to node.in_node(1).value.shape
# We will multiply weights according output/input channel dimension
ch_dim = weights_node.output_channel_dim if backward else weights_node.input_channel_dim
shape = np.array([weights_node.shape[ch_dim]])
# Scalar broadcast
if value.size == 1:
value = np.full(shape, value.item())
# Common broadcast for forward fusion
if not backward:
cnt = shape[-1] / value.shape[0]
if fuse_node.layout == 'NCHW':
tmp = []
for val in value:
tmp = np.concatenate((tmp, np.repeat(val, cnt)))
value = np.array(tmp)
else:
value = np.tile(value, int(cnt))
# Expand dims for multiplication (ex. [38] to [38, 1, 1])
wdims_number = weights_node.dims_number
for x in range(wdims_number - ch_dim - 1):
shape = np.append(shape, 1)
mul_val = np.array(value)
value = np.reshape(value, shape)
# Weights multiplication
weights_node.value = weights_node.value * value
# If we fuse in backward direction we should multiply biases if they exists
if backward and len(fuse_node.in_nodes()) == 3:
conv_bias = fuse_node.in_node(2)
conv_bias.value = conv_bias.value * np.squeeze(mul_val)
log.debug('Fused: {} to {}'.format(node.name, fuse_node.name))
is_fused = True
if is_fused:
# Delete Mul node
out_node = node.out_node()
op_data_node = node.in_node(tensor_id)
op_const_node = node.in_node(const_id)
op_node = op_data_node.in_node(0)
graph.remove_edge(node.id, out_node.id)
graph.remove_edge(op_node.id, op_data_node.id)
graph.remove_edge(op_const_node.id, node.id)
# Connect nodes after deleting
graph.add_edge(op_node.id, out_node.id, out=0)
for idx in reversed(range(len(op_data_node.out_nodes()))):
out_data = op_data_node.out_nodes()[idx]
edge_attrs = graph.get_edge_data(op_data_node.id, out_data.id)[0]
if not out_data.id is node.id:
graph.remove_edge(op_data_node.id, out_data.id)
graph.add_edges_from([(out_node.id, out_data.id, edge_attrs)])
return is_fused
def _fuse_add(graph: nx.MultiDiGraph,
node: Node,
fuse_nodes: list,
backward: bool = True):
"""
This function takes Add node and Convolution/FC nodes for further fusion and then deletes Add node
In case if Convolution/FC Bias absence it will be created
"""
is_fused = False
const_id, tensor_id = get_value_id(node), get_tensor_id(node)
if const_id is None or tensor_id is None:
log.warning(
'Cannot do fuse_add for node {} because this node has wrong inputs'
.format(node.id))
return False
# if len(node.in_node(const_id).shape) > 2 or any([x == 0 for x in node.in_node(const_id).shape]):
# log.warning('Cannot do fuse_add for node {} because this node has wrong shape'.format(node.id))
# return False
for fuse_node in fuse_nodes:
if fuse_node.soft_get('can_be_fused') == False:
log.warning(
'Node {} can\'t be used in fusing due to user specified attr can_be_fused = False'
.format(fuse_node.id))
return False
if not fuse_node.has_valid('layout'):
log.warning('Node {} has no layout attr'.format(fuse_node.id))
return False
if len(fuse_node.in_nodes()) < 2:
log.warning('Node {} has no weights node'.format(fuse_node.id))
return False
for fuse_node in fuse_nodes:
value = np.array(node.in_node(const_id).value)
# If forward, broadcast value
if not backward:
cnt = fuse_node.in_node(1).shape[-1] / node.in_node(
const_id).shape[0]
if fuse_node.layout == 'NCHW':
tmp = []
for val in value:
tmp = np.concatenate((tmp, np.repeat(val, cnt)))
value = np.array(tmp)
else:
value = np.tile(value, int(cnt))
value = np.squeeze(value)
# Create BIAS data node if not exists
if len(fuse_node.in_nodes()) <= 2:
bias_data = unique_id(graph, "bias_data")
data_type = fuse_node.in_node(1).data_type
# Broadcast if scalar
if value.size == 1:
id = fuse_node.in_node(
1).output_channel_dim if backward else fuse_node.in_node(
1).input_channel_dim
vshape = fuse_node.in_node(1).shape[id]
value = np.full(vshape, value.item())
if not backward:
value = np.dot(fuse_node.in_node(1).value, value)
shape = value.shape
graph.add_node(
bias_data,
**add_attrs_props(
dict(kind='data',
precision="FP32",
name=bias_data,
value=value,
shape=shape,
data_type=data_type)))
graph.add_edges_from([(bias_data, fuse_node.id, {
'in': 2,
'bin': 'biases'
})])
fuse_node['bias_term'] = True
else:
if not backward:
fuse_node.in_node(2).value += np.dot(
fuse_node.in_node(1).value, value)
else:
fuse_node.in_node(2).value += value
log.debug('Fused: {} to {}'.format(node.name, fuse_node.name))
is_fused = True
if is_fused:
# Delete Add node
out_node = node.out_node()
op_data_node = node.in_node(tensor_id)
op_const_node = node.in_node(const_id)
op_node = op_data_node.in_node(0)
graph.remove_edge(node.id, out_node.id)
graph.remove_edge(op_node.id, op_data_node.id)
graph.remove_edge(op_const_node.id, node.id)
# Connect nodes after deleting
graph.add_edge(op_node.id, out_node.id, out=0)
for idx in reversed(range(len(op_data_node.out_nodes()))):
out_data = op_data_node.out_nodes()[idx]
edge_attrs = graph.get_edge_data(op_data_node.id, out_data.id)[0]
if not out_data.id is node.id:
graph.remove_edge(op_data_node.id, out_data.id)
graph.add_edges_from([(out_node.id, out_data.id, edge_attrs)])
return is_fused
def muladd_to_scaleshift_action(graph: nx.MultiDiGraph, match: dict):
mul = match['mul']
add = match['add']
output = match['output']
if mul.soft_get('can_be_scaleshift') is False or add.soft_get(
'can_be_scaleshift') is False:
return
mul_weights_id = get_value_id(mul)
mul_input_id = get_tensor_id(mul)
add_weights_id = get_value_id(add)
if mul_weights_id is None:
log.debug("Mul->Add to ScaleShift: Mul {} has no weights".format(
mul.name))
return
if mul_input_id is None:
log.debug("Mul->Add to ScaleShift: Mul {} has no input".format(
mul.name))
return
if add_weights_id is None:
log.debug("Mul->Add to ScaleShift: Add {} has no weights".format(
add.name))
return
input = mul.in_node(mul_input_id)
weights = mul.in_node(mul_weights_id)
bias = add.in_node(add_weights_id)
# Transform values
weights.value = np.squeeze(weights.value)
weights.shape = weights.value.shape
bias.value = np.squeeze(bias.value)
bias.shape = bias.value.shape
# Broadcast weights if they are scalar
if weights.value.ndim == 0 and bias.value.ndim == 1:
weights.value = np.full(bias.shape, weights.value.item())
weights.shape = weights.value.shape
if bias.shape != weights.shape:
log.warning('Mul->Add to ScaleShift conversion stoped {} != {}'.format(
weights.shape, bias.shape))
return
if bias.value.ndim != weights.value.ndim or bias.value.size != weights.value.size:
log.debug(
"Skipping Mul->Add to scaleshift or power conversion for nodes {}, {} because of different weights "
"and biases".format(mul.name, add.name))
return
op_name = "ScaleShift"
if bias.value.size == 1 and weights.value.size == 1:
op_name = "Power"
log.debug(
"Fusing Mul->Add to {}. Input nodes: {} and {}, bias.shape = {}, weights.shape = {}"
"".format(op_name, mul.id, add.id, bias.shape, weights.shape))
graph.remove_edge(input.node, mul.id)
graph.remove_edge(weights.node, mul.id)
graph.remove_edge(bias.node, add.id)
graph.remove_edge(add.node, output.id)
op_node = unique_id(graph, mul.name + '/Fused{}_'.format(op_name))
if op_name == 'ScaleShift':
graph.add_node(
op_node,
**add_attrs_props(
dict(kind='op',
precision="FP32",
type=op_name,
name=op_node,
op=op_name,
data_type=input.data_type)))
update_ie_fields(graph.node[op_node])
graph.add_edges_from([(input.node, op_node, {
'in': 0
}), (weights.node, op_node, {
'in': 1,
'bin': 'weights'
}), (bias.node, op_node, {
'in': 2,
'bin': 'biases'
}), (op_node, output.node, {
'out': 0
})])
else:
graph.add_node(
op_node,
**add_attrs_props(
dict(kind='op',
precision="FP32",
type=op_name,
name=op_node,
op=op_name,
data_type=input.data_type,
power=1,
scale=weights.value.item(),
shift=bias.value.item())))
update_ie_fields(graph.node[op_node])
graph.add_edges_from([(input.node, op_node, {
'in': 0
}), (op_node, output.node, {
'out': 0
})])
return