def unsqueeze_num_directions(graph: Graph, match: dict):
""" Assuming considered LSTM/GRU/RNN node should has num_directions in output shape and add Unsqueeze
to match it.
"""
rnn_layer = match['rnn_layer']
rnn_layer_name = rnn_layer.soft_get('name', rnn_layer.id)
# num_directions is at 1st position in output shape, and in 0st position in hidden and cell states
# please refer to docs in this transform
direction_dim = [1, 0, 0] # index of dimension with direction index
for i in rnn_layer.out_nodes():
old_data_node = rnn_layer.out_node(i)
old_shape = old_data_node.shape.copy()
new_shape = shape_delete(old_shape, direction_dim[i])
data = Op._create_data_node(graph, name=rnn_layer.name + '/Out/{}/'.format(i), attrs={'shape': new_shape})
graph.remove_edge(rnn_layer.id, old_data_node.id)
graph.add_edge(rnn_layer.id, data.id, key=0, out=i)
unsqueeze = Unsqueeze(graph, dict())
unsqueeze_dim_data = Const(graph, {'name': rnn_layer.name + '/UnsqueezeNumDirections/{}/Dim'.format(i),
'value': int64_array([direction_dim[i]])}).create_node_with_data()
unsqueeze.create_node_with_data([data, unsqueeze_dim_data],
dict(name=rnn_layer_name + '/UnsqueezeNumDirections/{}'.format(i)),
data_nodes=[old_data_node])
def infer(node: Node):
"""
Performs shape inference of MatMul node as operation doc-string says
Raises on any shape inconsistency
"""
name = node.soft_get('name', str(node.id))
connected_in_ports = {
idx: port
for idx, port in node.in_ports().items()
if not port.disconnected()
}
assert len(connected_in_ports) == 2 and 0 in connected_in_ports and 1 in connected_in_ports, \
"MatMul should have 2 connected input ports, but it doesn't for node: `{}`. Ports: {}" \
"".format(name, connected_in_ports)
log.debug('MatMul `{}` input shapes: {}'.format(
name, [node.in_port(i).data.get_shape() for i in range(2)]))
A_shape, B_shape = MatMul.shape_alignment(node)
log.debug('MatMul `{}` aligned input shapes: {}'.format(
name, [A_shape, B_shape]))
assert compatible_dims(A_shape[-1], B_shape[-2]), \
"MatMul input shapes are incorrect. COL_INDEX_DIMs are not equal. Node: {}. Shapes: {}" \
"".format(name, [A_shape, B_shape])
output_shape = np.ma.concatenate((A_shape[:-1], B_shape[-1:]))
if node.in_port(0).data.get_shape().size == 1:
assert compatible_dims(output_shape[-2], 1)
output_shape = shape_delete(output_shape, -2)
if node.in_port(1).data.get_shape().size == 1:
assert compatible_dims(output_shape[-1], 1)
output_shape = shape_delete(output_shape, -1)
node.out_port(0).data.set_shape(output_shape)
in_ch = 0 if not node.transpose_b else 1
out_ch = 1 if not node.transpose_b else 0
assign_dims_to_weights(node.in_node(1), None, in_ch, out_ch,
node.in_port(1).data.get_shape().size)
MatMul.value_propagation(node)
def infer(node: Node):
real_squeeze_dims = int64_array([])
input_shape = node.in_port(0).data.get_shape()
node_name = node.soft_get('name', node.id)
if input_shape is None:
raise Error(
'Input shape is not defined for node {}'.format(node_name))
output_shape = input_shape.copy()
assert len(node.in_nodes(
)) == 2, 'The Squeeze node {} must have 2 inputs'.format(node_name)
# TODO remove the following 'if' statement when IE start support 0D tensors
squeeze_dims = node.in_port(1).data.get_value()
if squeeze_dims.ndim == 0:
squeeze_dims = squeeze_dims.reshape([1])
for dim in squeeze_dims:
if output_shape[dim] == 1 or output_shape[dim] is dynamic_dimension:
real_squeeze_dims = np.ma.append(
real_squeeze_dims,
get_canonical_axis_index(output_shape, dim))
else:
raise Error(
'Trying to squeeze dimension not equal to 1 for node "{}"'.
format(node_name))
# if squeeze_dims empty then all 1s should be removed (tf specification of Squeeze op)
if squeeze_dims.size == 0:
for i in range(output_shape.size):
if output_shape[i] == 1:
real_squeeze_dims = np.ma.append(
real_squeeze_dims,
get_canonical_axis_index(output_shape, i))
assert is_fully_defined(
real_squeeze_dims
), 'Squeeze dimension(s) is not defined for op "{}"'.format(node_name)
output_shape = shape_delete(output_shape, real_squeeze_dims)
node.out_port(0).data.set_shape(output_shape)
# make dimensions positive to correctly translate from NHWC to NCHW layout
if node.in_port(1).get_source().node.op == 'Const':
node.in_port(1).data.set_value(real_squeeze_dims)
if node.in_port(0).data.get_value() is not None:
node.out_port(0).data.set_value(
node.in_port(0).data.get_value().reshape(output_shape))
# the squeeze_dim attribute will be converted to the second input in the end of the Middle phase
PermuteInputs().set_input_permutation(node.in_node(1), node, 'input:0',
'axis')
def test_shape_delete_raise_exception(self):
with self.assertRaisesRegex(Error, '.*Incorrect parameter type.*'):
shape_delete(gen_masked_array([1, 2, 3], []), {})
def test_shape_delete(self, shape, indices, result):
self.assertTrue(
strict_compare_tensors(shape_delete(shape, indices), result))
def infer(node):
name = node.soft_get('name', node.id)
op = node.soft_get('op', None)
assert op is not None and op in ['VariadicSplit', 'AttributedVariadicSplit'], \
'Unexpected `op`={} attribute for Split-like node {}'.format(op, name)
num_in_ports = 1 if op == 'AttributedVariadicSplit' else 3 if op == 'VariadicSplit' else None
assert num_in_ports in [1, 3], \
'VariadicSplitBase supports AttributedVariadicSplit with 1 input and VariadicSplit with 3 inputs, ' \
'but it is {} for {} node {}'.format(num_in_ports, op, name)
connected_inputs = {
idx: port
for idx, port in node.in_ports().items()
if not port.disconnected()
}
assert len(connected_inputs) == num_in_ports and all([i in connected_inputs for i in range(num_in_ports)]), \
"{} should have {} connected input ports, but it doesn't for node: `{}`. Ports: {}" \
"".format(op, num_in_ports, name, connected_inputs)
input_shape = node.in_port(0).data.get_shape()
assert input_shape is not None
axis = node.in_port(1).data.get_value(
) if op == 'VariadicSplit' else node.soft_get('axis', None)
assert axis is not None, '{} `axis` is unknown for node {}'.format(
op, name)
assert axis.ndim == 0 or (axis.ndim == 1 and axis.shape[0] == 1), \
'{} `axis` should be scalar or tensor with shape [1], but it`s not for node {}'.format(op, name)
split_lengths = node.in_port(2).data.get_value(
) if op == 'VariadicSplit' else node.soft_get('split_lengths', None)
assert split_lengths is not None, '{} `split_lengths` is unknown for node {}'.format(
op, name)
undefined_elements = np.argwhere(split_lengths == -1).flatten()
assert undefined_elements.size <= 1, \
'{} split_lengths=`{}` is a list with output sizes, only one of which could be -1. Node: {}' \
''.format(op, split_lengths, name)
input_elements = input_shape[axis]
assert undefined_elements.size != 0 or input_elements is dynamic_dimension or \
input_elements == np.sum(split_lengths), 'The sum of split_lengths=`{}` must match data.shape[axis]=' \
'`{}`. Node: {}'.format(split_lengths, input_elements, name)
assert len(split_lengths) >= len([port for i, port in node.out_ports().items() if not port.disconnected()]), \
'Number of split_lengths=`{}` is less than connected output ports. Node: {}'.format(split_lengths, name)
# in split_lengths some value can be 0, in this case we will ignore it:
# * remove according branch
# * remove 0 from split_lengths
for i in reversed(range(len(split_lengths))):
if split_lengths[i] == 0:
if node.out_port(i).disconnected():
split_lengths = shape_delete(split_lengths, i)
if op == 'VariadicSplit':
node.in_port(2).data.set_value(split_lengths)
else:
node['split_lengths'] = split_lengths
delete_out_port(i, node)
else:
log.error(
"Zero dimension on {} branch after Split node {}".
format(i, node.id))
return
# shape propagation
idxs, curr_pos = [], 0
for i, piece in enumerate(split_lengths):
assert piece >= -1, 'VariadicSplit split_lengths=`{}` should be non-negative'.format(
split_lengths)
out_shape = input_shape.copy()
split_length = piece if piece > -1 else input_elements - (
np.sum(split_lengths) + 1)
out_shape[axis] = split_length
curr_pos = curr_pos + split_length
idxs.append(curr_pos)
if not node.out_port(i).disconnected():
node.out_port(i).data.set_shape(out_shape)
# value propagation
input_value = node.in_port(0).data.get_value()
if input_value is not None:
split = np.split(input_value, idxs[:-1], axis)
for i, port in node.out_ports().items():
if not port.disconnected():
port.data.set_value(split[i])
if op == 'VariadicSplit':
PermuteInputs().set_input_permutation(node.in_node(1), node,
'input:0', 'axis')
elif op == 'AttributedVariadicSplit':
PermuteAttrs.create_permute_attrs(node,
attrs=[('axis', 'input:0')])
def replace_pattern(self, graph: Graph, match: dict):
lstm = match['lstm']
# Build TensorIterator body first
body = Graph(name=lstm.name + '/sub_graph')
body.graph = graph.graph
# 1. Input squeeze Reshape
inputs = [Op._create_data_node(body, lstm.name + '/inport/' + str(inp),
{'shape': lstm.in_node(inp).shape.copy(),
'value': lstm.in_node(inp).value.copy()
if lstm.in_node(inp).value is not None and inp in [1, 2] else None})
for inp in [0, 4, 5, 1, 2]] # X, WR, B, h_init, c_init
inputs[0].shape[lstm.sequence_dim] = 1
input_squeeze = Squeeze(body, dict(name=lstm.name + '/input_squeeze', internal_layer_id=0))
squeeze_dim_data = Const(body, {'name': lstm.name + '/input_squeeze_dim',
'value': [lstm.sequence_dim]}).create_node_with_data()
inputs[0] = input_squeeze.create_node_with_data([inputs[0], squeeze_dim_data],
edge_attrs=[{'internal_port_id': 0}])
# 2. Output unsqueeze Reshape
outputs = [Op._create_data_node(body, lstm.name + '/outport/' + str(out),
{'shape': lstm.out_node(out).shape.copy() if out in lstm.out_nodes()
else lstm.in_node(4).shape.copy()}) for out in [0, 1]]
for out in outputs:
add_opoutput(body, out.id, 0, False)
outputs[0].shape = shape_delete(outputs[0].shape, lstm.sequence_dim)
output_unsqueeze = Unsqueeze(body, dict(name=lstm.name + 'output_unsqueeze', internal_layer_id=2))
unsqueeze_dim_data = Const(body, {'name': lstm.name + '/output_unsqueeze_dim',
'value': [lstm.sequence_dim]}).create_node_with_data()
# 3. LSTMCell
lstm_cell_op = LSTMCell(body, dict(hidden_size=lstm.hidden_size,
activations=lstm.activations,
activation_alpha=lstm.activation_alpha,
activation_beta=lstm.activation_beta,
clip=lstm.clip,
input_forget=lstm.input_forget,
name=lstm.name + '/LSTMCell',
internal_layer_id=1))
lstm_cell_node = lstm_cell_op.create_node_with_data(inputs, data_nodes=outputs,
edge_attrs=[{}, {'internal_port_id': 1},
{'internal_port_id': 2}, {'bin': 'weights'},
{'bin': 'biases'}])
lstm_cell_node[0].in_node().out_edge(0)['internal_port_id'] = 4
lstm_cell_node[0].in_node().out_edge(1)['internal_port_id'] = 5
lstm_cell_node[0] = output_unsqueeze.create_node_with_data([lstm_cell_node[0], unsqueeze_dim_data])
lstm_cell_node[0].in_node().out_edge(0)['internal_port_id'] = 3
add_opoutput(body, lstm_cell_node[0].id, 0, False)
# 4. TensorIterator layer creating
assert lstm.direction in ['forward', 'reverse']
if lstm.direction == 'forward':
stride = 1
start = None
end = None
else:
assert lstm.direction == 'reverse'
stride = -1
start = -1
end = 0
output_port_map = [{
'external_port_id': 3,
'internal_layer_id': 2,
'internal_port_id': 3,
'axis': lstm.sequence_dim,
'stride': stride,
'start': start,
'end': end,
'part_size': 1,
}]
# Adding h_state, c_state to outputs
if len(lstm.out_nodes()) == 3:
output_port_map.extend([{
'external_port_id': 4,
'internal_layer_id': 1,
'internal_port_id': 4,
}, {
'external_port_id': 5,
'internal_layer_id': 1,
'internal_port_id': 5,
}])
ti_op = TensorIterator(graph, {
'name': lstm.name + '/TensorIterator',
'body': body,
'in_ports_count': 3,
'out_ports_count': len(lstm.out_nodes()),
'input_port_map': [
{
'external_port_id': 0,
'internal_layer_id': 0,
'internal_port_id': 0,
'axis': lstm.sequence_dim,
'stride': stride,
'start': start,
'end': end,
'part_size': 1,
},
{
'external_port_id': 1,
'internal_layer_id': 1,
'internal_port_id': 1,
},
{
'external_port_id': 2,
'internal_layer_id': 1,
'internal_port_id': 2,
},
],
'output_port_map': output_port_map,
'back_edges': [
{
'from_layer': 1,
'from_port': 4,
'to_layer': 1,
'to_port': 1,
},
{
'from_layer': 1,
'from_port': 5,
'to_layer': 1,
'to_port': 2,
},
]
})
assert sorted(lstm.out_nodes().keys()) == list(range(len(lstm.out_nodes()))), \
"There are gaps in output ports of LSTMSequence operation. Node {}".format(lstm.id)
outs = ti_op.create_node_with_data([lstm.in_node(i) for i in [0, 4, 5]], # X, h_init, c_init
data_nodes=[lstm.out_node(i) for i in range(len(lstm.out_nodes()))],
edge_attrs=[{'external_port_id': 0}, {'external_port_id': 1},
{'external_port_id': 2}])
if not isinstance(outs, list):
outs = list([outs])
graph.remove_node(lstm.id)
outs[0].in_edge(0)['external_port_id'] = 3
for i, out in enumerate(outs[1:]):
external_port_id = 4 + i
out.in_edge()['external_port_id'] = external_port_id
ti = outs[0].in_node()
TensorIterator.cover_body_input_data_nodes_with_parameter_ops(ti)
TensorIterator.cover_body_constant_data_nodes_with_const_ops(ti)
TensorIterator.normalize_internal_ids(ti)
def replace_pattern(self, graph: Graph, match: dict):
if match['rnn_layer']['op'] == 'LSTM':
return
rnn_layer = match['rnn_layer']
# Build TensorIterator body first
body = Graph(name=rnn_layer.name + '/sub_graph')
body.graph = graph.graph
# 1. Input squeeze Reshape
inputs = [
Op._create_data_node(
body, rnn_layer.name + '/inport/' + str(inp), {
'shape':
rnn_layer.in_node(inp).shape.copy(),
'value':
rnn_layer.in_node(inp).value.copy()
if rnn_layer.in_node(inp).value is not None
and inp in [1, 2] else None
}) for inp in [0, 4, 1, 2]
] # X, h_init, WR, B
inputs[0].shape[rnn_layer.sequence_dim] = 1
input_squeeze = Squeeze(
body,
dict(name=rnn_layer.name + '/input_squeeze', internal_layer_id=0))
input_squeeze_dim = Const(
body,
dict(name=rnn_layer.name + '/input_squeeze_dim',
value=rnn_layer.sequence_dim)).create_node_with_data()
inputs[0] = input_squeeze.create_node_with_data(
[inputs[0], input_squeeze_dim],
edge_attrs=[{
'internal_port_id': 0
}])
# 2. Output unsqueeze Reshape
outputs = [
Op._create_data_node(
body, rnn_layer.name + '/outport/' + str(out), {
'shape':
rnn_layer.out_node(out).shape.copy()
if out in rnn_layer.out_nodes() else None
}) for out in [0]
]
for out in outputs:
add_opoutput(body, out.id, 0, False)
outputs[0].shape = shape_delete(outputs[0].shape,
rnn_layer.sequence_dim)
output_unsqueeze_dim = Const(
body,
dict(name=rnn_layer.name + '/output_unsqueeze_dim',
value=rnn_layer.sequence_dim)).create_node_with_data()
output_unsqueeze = Unsqueeze(
body,
dict(name=rnn_layer.name + '/output_unsqueeze/',
internal_layer_id=2))
additional_attrs = dict(activations=rnn_layer.activations,
activation_alpha=rnn_layer.activation_alpha,
activation_beta=rnn_layer.activation_beta,
clip=rnn_layer.clip)
if rnn_layer.op == 'GRU':
additional_attrs[
'linear_before_reset'] = rnn_layer.linear_before_reset
# 3. ***Cell
rnn_cell_op = self.get_rnn_cell(rnn_layer['op'])(
body,
dict(hidden_size=rnn_layer.hidden_size,
name=rnn_layer.name + '/{}Cell'.format(rnn_layer.op),
**additional_attrs,
internal_layer_id=1))
gru_cell = rnn_cell_op.create_node_with_data(inputs,
data_nodes=outputs,
edge_attrs=[{}, {
'internal_port_id':
1
}, {
'internal_port_id':
2
}, {
'bin':
'weights'
}, {
'bin':
'biases'
}])
# internal ports for outputs of cell
gru_cell.in_node().out_edge(0)['internal_port_id'] = 4 # h_state
gru_cell = output_unsqueeze.create_node_with_data(
[gru_cell, output_unsqueeze_dim])
gru_cell.in_node().out_edge(0)['internal_port_id'] = 3
add_opoutput(body, gru_cell.id, 0, False)
# 4. TensorIterator layer creating
assert rnn_layer.direction in ['forward', 'reverse']
if rnn_layer.direction == 'forward':
stride = 1
start = None
end = None
else:
assert rnn_layer.direction == 'reverse'
stride = -1
start = -1
end = 0
# stacked h_state
output_port_map = [{
'external_port_id': 3,
'internal_layer_id': 2,
'internal_port_id': 3,
'axis': rnn_layer.sequence_dim,
'stride': stride,
'start': start,
'end': end,
'part_size': 1,
}]
# Adding last h_state to outputs
if len(rnn_layer.out_nodes()) == 2:
output_port_map.extend([{
'external_port_id': 4,
'internal_layer_id': 1,
'internal_port_id': 4,
}])
ti_op = TensorIterator(
graph,
{
'name':
rnn_layer.name + '/TensorIterator',
'body':
body,
'in_ports_count':
4,
'out_ports_count':
len(rnn_layer.out_nodes()),
'input_port_map': [
{
'external_port_id': 0,
'internal_layer_id': 0,
'internal_port_id': 0,
'axis': rnn_layer.sequence_dim,
'stride': stride,
'start': start,
'end': end,
'part_size': 1,
},
{
'external_port_id': 1,
'internal_layer_id': 1,
'internal_port_id': 1,
},
],
'output_port_map':
output_port_map,
# only for h state
'back_edges': [
{
'from_layer': 1,
'from_port': 4,
'to_layer': 1,
'to_port': 1,
},
]
})
assert sorted(rnn_layer.out_nodes().keys()) == list(range(len(rnn_layer.out_nodes()))), \
"There are gaps in output ports of GRUSequence operation. Node {}".format(rnn_layer.id)
outs = ti_op.create_node_with_data(
[rnn_layer.in_node(i) for i in [0, 4]], # X, h_init
data_nodes=[
rnn_layer.out_node(i)
for i in range(len(rnn_layer.out_nodes()))
],
edge_attrs=[{
'external_port_id': 0
}, {
'external_port_id': 1
}])
if not isinstance(outs, list):
outs = list([outs])
graph.remove_node(rnn_layer.id)
outs[0].in_edge(0)['external_port_id'] = 3
for i, out in enumerate(outs[1:]):
external_port_id = 4 + i
out.in_edge()['external_port_id'] = external_port_id
ti = outs[0].in_node()
TensorIterator.cover_body_input_data_nodes_with_parameter_ops(ti)
TensorIterator.cover_body_constant_data_nodes_with_const_ops(ti)
TensorIterator.normalize_internal_ids(ti)
