def infer(node: Node):
node_name = node.soft_get('name', node.id)
connected_in_ports = [
port for port in node.in_ports().values()
if not port.disconnected()
]
assert len(connected_in_ports) == 2, \
"Incorrect number of inputs for {} node".format(node_name)
logits_shape = node.in_port(0).data.get_shape()
sequence_mask_shape = node.in_port(1).data.get_shape()
# check shapes of input tensors
assert len(logits_shape) == 3, \
'Incorrect rank of logits for {} node'.format(node_name)
assert len(sequence_mask_shape) == 2, \
'Incorrect rank of sequence length tensor for {} node'.format(node_name)
assert compatible_dims(logits_shape[1], sequence_mask_shape[1]), \
'Batch dimensions of input tensors must be the same for {} node'.format(node_name)
assert compatible_dims(logits_shape[0], sequence_mask_shape[0]), \
'Time dimensions of input tensors must be the same for {} node'.format(node_name)
batch_size = logits_shape[1]
time_size = logits_shape[0]
node.out_port(0).data.set_shape([batch_size, time_size, 1, 1])
def infer(node: Node):
if node.has_and_set('extra_inputs'):
assert len(node.in_nodes()) == 8
else:
assert len(node.in_nodes()) == 5
assert len(node.out_nodes()) in [1, 2]
hidden_shape = node.in_node(1).shape.copy()
cell_shape = node.in_node(2).shape.copy()
mark_input_bins(node, start_port=3)
node.out_node(0).shape = hidden_shape
if len(node.out_nodes()) == 2:
node.out_node(1).shape = cell_shape
hidden_size = hidden_shape[1]
if node.has_valid('hidden_size'):
if node.hidden_size != hidden_size:
raise Error(
"Input shape {} for hidden size doesn't match pre-defined hidden_size in node {}"
.format(node.in_node(1).shape, node.soft_get('name')))
else:
node['hidden_size'] = hidden_size
assert cell_shape[1] == hidden_size
input_shape = node.in_node(0).shape
assert input_shape is not None
assert compatible_dims(hidden_shape[0], cell_shape[0]) and \
compatible_dims(cell_shape[0], input_shape[0]), 'States are not broadcast-able by batch for node {}' \
''.format(node.soft_get('name', node.id))
def multi_box_detection_infer(node: Node):
loc_shape = node.in_node(0).shape
conf_shape = node.in_node(1).shape
prior_boxes_shape = node.in_node(2).shape
node_name = node.soft_get('name', node.id)
if loc_shape is None or conf_shape is None or prior_boxes_shape is None:
raise Error(
'Shapes for the Detection Output node "{}" are not defined'.format(
node_name))
prior_size = 4
if node.has('normalized') and not node.normalized:
prior_size = 5
if is_fully_defined(
prior_boxes_shape[-1]) and prior_boxes_shape[-1] % prior_size != 0:
raise Error(
'Amount of confidences "{}" is not divisible by {} for node "{}"'
''.format(prior_boxes_shape[-1], prior_size, node_name))
num_priors = prior_boxes_shape[-1] // prior_size
if not node.has_valid('keep_top_k') or node.keep_top_k == -1:
node['keep_top_k'] = num_priors
# do not try to infer number of classes because it is not possible in case when input shapes are partially defined
if not node.has_valid('num_classes'):
node['num_classes'] = conf_shape[-1] // num_priors
log.debug('Inferred amount of classes "{}"'.format(node.num_classes))
num_loc_classes = node.num_classes
if node.has_and_set('share_location') and node.share_location:
num_loc_classes = 1
if not compatible_dims(num_priors * num_loc_classes * 4, loc_shape[-1]):
raise Error(
'Locations and prior boxes shapes mismatch: "{}" vs "{}" for node "{}"'
''.format(loc_shape, prior_boxes_shape, node_name))
if not node.variance_encoded_in_target and not compatible_dims(
prior_boxes_shape[-2], 2):
raise Error(
'The "-2" dimension of the prior boxes must be 2 but it is "{}" for node "{}".'
''.format(prior_boxes_shape[-2], node_name))
if is_fully_defined(conf_shape[-1]) and is_fully_defined(
num_priors) and conf_shape[-1] % num_priors != 0:
raise Error(
'Amount of confidences "{}" is not divisible by amount of priors "{}" for node "{}".'
''.format(conf_shape[-1], num_priors, node_name))
node.out_port(0).data.set_shape([1, 1, conf_shape[0] * node.keep_top_k, 7])
# the line below is needed for the TF framework so the MO will not change the layout
node.graph.node[node.out_node(0).id]['nchw_layout'] = True
def infer(node: Node):
node_name = node.soft_get('name', node.id)
connected_in_ports = [
port for port in node.in_ports().values()
if not port.disconnected()
]
assert len(connected_in_ports) in [2, 3], \
"Incorrect number of inputs for {} node".format(node_name)
logits_shape = node.in_port(0).data.get_shape()
sequence_len_shape = node.in_port(1).data.get_shape()
if len(node.in_nodes()) == 3:
blank_index_shape = node.in_port(2).data.get_shape()
assert len(blank_index_shape) == 1, \
'Incorrect rank of blank_index for {} node'.format(node_name)
# check shapes of input tensors
assert len(logits_shape) == 3, \
'Incorrect rank of logits for {} node'.format(node_name)
assert len(sequence_len_shape) == 1, \
'Incorrect rank of sequence length tensor for {} node'.format(node_name)
assert compatible_dims(logits_shape[0], sequence_len_shape[0]), \
'Batch dimensions of input tensors must be the same for {} node'.format(node_name)
batch_size = logits_shape[0]
time_size = logits_shape[1]
if node.is_out_port_connected(0):
node.out_port(0).data.set_shape([batch_size, time_size])
if node.is_out_port_connected(1):
node.out_port(1).data.set_shape([batch_size])
def infer(node: Node):
name = node.soft_get('name', node.id)
connected_in_ports = {
idx: port
for idx, port in node.in_ports().items()
if not port.disconnected()
}
assert len(connected_in_ports) >= 2 and 0 in connected_in_ports and 1 in connected_in_ports, \
'FullyConnected should have 2 connected input ports, but it doesn\'t for node: `{}`. Ports: {}' \
''.format(name, connected_in_ports)
assert node.has_valid('out-size')
input_shape = node.in_port(0).data.get_shape()
weights_shape = node.in_port(1).data.get_shape()
assert input_shape is not None and weights_shape is not None, \
'Incorrect FullyConnected input shapes. Node: {}. Shapes: {}'.format(name, [input_shape, weights_shape])
assert weights_shape.size == 2
out_size = node.soft_get('out-size')
assert compatible_dims(weights_shape[0], out_size), \
'weights_shape={}, out-size={}'.format(weights_shape, out_size)
if 2 in connected_in_ports:
bias_value = node.in_port(2).data.get_value()
bias_shape = node.in_port(2).data.get_shape()
assert bias_shape is not None, 'Shape was not inferred for biases of FullyConnected {}'.format(
name)
assert bias_value is not None, 'Value was not inferred for biases of FullyConnected {}'.format(
name)
assert compatible_shapes(bias_shape, [out_size]) or compatible_shapes(bias_shape, [1, out_size]), \
'Incorrect FullyConnected bias shape `{}` for node {}. `out-size`={}'.format(bias_shape, node, out_size)
node.out_port(0).data.set_shape([*input_shape[:-1], out_size])
def infer(node: Node):
"""
Performs shape inference of MatMul node as operation doc-string says
Raises on any shape inconsistency
"""
name = node.soft_get('name', str(node.id))
connected_in_ports = {
idx: port
for idx, port in node.in_ports().items()
if not port.disconnected()
}
assert len(connected_in_ports) == 2 and 0 in connected_in_ports and 1 in connected_in_ports, \
"MatMul should have 2 connected input ports, but it doesn't for node: `{}`. Ports: {}" \
"".format(name, connected_in_ports)
log.debug('MatMul `{}` input shapes: {}'.format(
name, [node.in_port(i).data.get_shape() for i in range(2)]))
A_shape, B_shape = MatMul.shape_alignment(node)
log.debug('MatMul `{}` aligned input shapes: {}'.format(
name, [A_shape, B_shape]))
assert compatible_dims(A_shape[-1], B_shape[-2]), \
"MatMul input shapes are incorrect. COL_INDEX_DIMs are not equal. Node: {}. Shapes: {}" \
"".format(name, [A_shape, B_shape])
output_shape = np.ma.concatenate((A_shape[:-1], B_shape[-1:]))
if node.in_port(0).data.get_shape().size == 1:
assert compatible_dims(output_shape[-2], 1)
output_shape = shape_delete(output_shape, -2)
if node.in_port(1).data.get_shape().size == 1:
assert compatible_dims(output_shape[-1], 1)
output_shape = shape_delete(output_shape, -1)
node.out_port(0).data.set_shape(output_shape)
in_ch = 0 if not node.transpose_b else 1
out_ch = 1 if not node.transpose_b else 0
assign_dims_to_weights(node.in_node(1), None, in_ch, out_ch,
node.in_port(1).data.get_shape().size)
MatMul.value_propagation(node)
def infer(node):
layout = node.graph.graph['layout']
node_name = node.soft_get('name', node.id)
assert len([port for port in node.in_ports().values() if not port.disconnected()]) == 3, \
'The node "{}" must 3 inputs'.format(node_name)
assert node.has_valid(
'pooled_w'
), '"pooled_w" attribute is not set for node "{}"'.format(node_name)
assert node.has_valid(
'pooled_h'
), '"pooled_h" attribute is not set for node "{}"'.format(node_name)
assert node.has_valid(
'mode'), '"mode" attribute is not set for node "{}"'.format(
node_name)
assert node.mode in ['avg', 'max'], \
'"mode" attribute range of values is ["avg", "max"], got {} for node "{}"'.format(node.mode, node_name)
input_shape = node.in_port(0).data.get_shape()
rois_shape = node.in_port(1).data.get_shape()
indices_shape = node.in_port(2).data.get_shape()
assert input_shape is not None and rois_shape is not None and indices_shape is not None, \
'The node "{}" input shape is None'.format(node_name)
assert compatible_dims(rois_shape[0], indices_shape[0]), 'The number of batch indices does not correspond ' \
'to number of ROIs for node "{}"'.format(node_name)
assert compatible_dims(
rois_shape[1],
4), 'The size of ROI element must be 4 for node "{}"'.format(
node_name)
assert len(
input_shape
) == 4, 'The rank of port 0 input tensor of node "{}" must be 4.'.format(
node_name)
node.out_port(0).data.set_shape(
shape_for_layout(layout,
batch=rois_shape[0],
features=input_shape[get_features_dim(layout, 4)],
height=node.pooled_h,
width=node.pooled_w))
def insert_pre_processing(graph: Graph, input_node: Node,
node_mean_scale_values: np.array,
preprocessing_name: str):
assert preprocessing_name in ['scale', 'mean']
if node_mean_scale_values.get(preprocessing_name) is None:
return
user_value = node_mean_scale_values[preprocessing_name]
value = 1 / user_value if preprocessing_name == 'scale' else user_value * (
-1)
optimize_value = int(preprocessing_name == 'scale')
op = Mul if preprocessing_name == 'scale' else Add
if all([x == optimize_value for x in value]):
return
assert input_node.has_valid('shape')
features_dim_idx = get_features_dim(graph.graph['layout'],
len(input_node.shape))
assert compatible_dims(
value.size, input_node.shape[features_dim_idx]) or value.size == 1
shape = np.ones(len(input_node.shape), dtype=np.int64)
shape[features_dim_idx] = value.size
value = value.reshape(shape)
name = input_node.soft_get('name',
input_node.id) + '/' + preprocessing_name
preprocessing = create_op_with_const_inputs(graph,
op=op,
port_value_dict={1: value},
op_attrs={'name': name})
for dst in input_node.out_port(0).get_destinations():
if dst.node.soft_get('type') != 'ShapeOf':
# After the insertion of additional operations model optimizer
# should keep the link to the input layer. Parameter node in framework
# should map to parameter node in IR.
# For this reason 'fw_tensor_debug_info' should be kept in data node.
dst.get_connection().set_source(preprocessing.out_port(0),
"source")
input_node.out_port(0).connect(preprocessing.in_port(0))
def replace_pattern(graph, match: dict):
# Check for SS params
# Sanity check that we iterate over axis of some tensor
ss = match['Strided_slice']
params = ss.in_nodes()
assert np.all(params[1].in_node().value == 0)
assert np.all(params[2].in_node().value == 1)
assert np.all(params[3].in_node().value == 1)
# Check for comparing SS and seq_length source (it should be one tensor)
# SIMPLE CHECK
assert match['Strided_slice_data'].value is not None
if match['minimum_data'].value is None:
log.warning(
'TF loop doesn\'t have a constant upper bound produced by node {}, or ModelOptimizer '
'cannot detect a constant in this case. Loops with a dynamic number of iterations are not '
'supported, so in the resulting IR, generated TensorIterator will have '
'a maximum number of iterations determined by input tensor size: {}'
''.format(match['minimum_data'].soft_get('name'),
match['Strided_slice_data'].value))
else:
assert compatible_dims(match['Strided_slice_data'].value, match['minimum_data'].value), \
'Values do not match: {} and {}'.format(match['Strided_slice_data'].value, match['minimum_data'].value)
# Check that bound for Condition and Inputs/Outputs sizes match
condition_time = match['condition'].out_node(0)
inputs_and_outputs = condition_time.out_nodes()
type_list = ['TensorIteratorInput']
for ta in inputs_and_outputs:
if ta.has_valid(
'kind') and ta['kind'] == 'op' and ta['op'] in type_list:
assert ta.in_node(0).id == ss.id
log.debug(
'+++++++++++++++ Condition Check was successful ++++++++++++++++')
def infer(node: Node):
node_name = node.soft_get('name', node.id)
connected_in_ports = [port for port in node.in_ports().values() if not port.disconnected()]
assert len(connected_in_ports) == 2, \
"Incorrect number of inputs for {} node".format(node_name)
data_shape = node.in_port(0).data.get_shape()
data_value = node.in_port(0).data.get_value()
indices_shape = node.in_port(1).data.get_shape()
indices_value = node.in_port(1).data.get_value()
assert node.has_valid('batch_dims'), "Node {} must contain `batch_dims` attribute".format(node_name)
batch_dims = node.batch_dims
# check that a number of batch dimensions is less than both ranks of data and indices tensors
assert batch_dims < len(data_shape), "Number of batch dimensions must be less than a rank of data"
assert batch_dims < len(indices_shape), "Number of batch dimensions must be less than a rank of indices"
# check that batch dimensions of data and indices are the same
for batch_dim in range(batch_dims):
assert compatible_dims(data_shape[batch_dim], indices_shape[batch_dim]), \
"The dimension {} for data and indices tensors must be the same".format(batch_dim)
# check ranks of input tensors
assert len(data_shape) > 0, "Data must not be a scalar"
assert len(indices_shape) > 0, "Indices must not be a scalar"
assert (batch_dims + indices_shape[-1]) <= len(data_shape), \
"Length of a tuple with indices must not exceed a rank of data tensor excluding batch dimensions"
assert node['version'] in ['opset5', 'opset8'], 'Unsupported version of GatherND operation: {}, operation ' \
'name : {}'.format(node['version'], node.soft_get('name'))
# compute output shape
batch = []
if batch_dims > 0:
if node['version'] == 'opset5': # Support old version of gatherND shape inference
if is_fully_defined(data_shape[:batch_dims]):
batch = [np.prod(data_shape[:batch_dims]).tolist()]
else:
batch = [dynamic_dimension_value]
elif node['version'] == 'opset8':
for dim in range(batch_dims):
assert compatible_dims(indices_shape[dim], data_shape[dim]),\
"Batch dimensions in data.shape and indices.shape must be compatible"
if is_fully_defined(indices_shape[:batch_dims]):
batch = indices_shape[:batch_dims].tolist()
elif is_fully_defined(data_shape[:batch_dims]):
batch = data_shape[:batch_dims].tolist()
else:
for ind in range(batch_dims):
if indices_shape[ind] != dynamic_dimension_value:
batch.append(indices_shape[ind])
elif data_shape[ind] != dynamic_dimension_value:
batch.append(data_shape[ind])
else:
batch.append(dynamic_dimension_value)
slice_shape = list(data_shape[(batch_dims + indices_shape[-1]):])
output_shape = batch + list(indices_shape)[batch_dims:-1] + slice_shape
node.out_port(0).data.set_shape(output_shape)
# compute output value if all input indices are defined
if is_fully_defined(indices_value) and data_value is not None:
batch_dims_size = 1
for i in range(batch_dims):
batch_dims_size *= indices_shape[i]
output_data = []
reshaped_indices = indices_value.reshape(batch_dims_size, -1, indices_shape[-1])
reshaped_data = data_value.reshape((batch_dims_size,) + tuple((data_shape[batch_dims:])))
for batch_dim in range(reshaped_indices.shape[0]):
for outer_dim in range(reshaped_indices.shape[1]):
gather_index = tuple(reshaped_indices[batch_dim][outer_dim])
output_data.append(reshaped_data[(batch_dim,) + gather_index])
output_value = np.asarray(output_data, dtype=data_value.dtype).reshape(output_shape)
node.out_port(0).data.set_value(output_value)
def infer(node: Node):
node_name = node.soft_get('name', node.id)
connected_in_ports = [
port for port in node.in_ports().values()
if not port.disconnected()
]
assert len(connected_in_ports) == 2, \
"Incorrect number of inputs for {} node".format(node_name)
data_shape = node.in_port(0).data.get_shape()
data_value = node.in_port(0).data.get_value()
indices_shape = node.in_port(1).data.get_shape()
indices_value = node.in_port(1).data.get_value()
assert node.has_valid(
'batch_dims'
), "Node {} must contain `batch_dims` attribute".format(node_name)
batch_dims = node.batch_dims
# check that a number of batch dimensions is less than both ranks of data and indices tensors
assert batch_dims < len(
data_shape
), "Number of batch dimensions must be less than a rank of data"
assert batch_dims < len(
indices_shape
), "Number of batch dimensions must be less than a rank of indices"
# check that batch dimensions of data and indices are the same
for batch_dim in range(batch_dims):
assert compatible_dims(data_shape[batch_dim], indices_shape[batch_dim]), \
"The dimension {} for data and indices tensors must be the same".format(batch_dim)
# check ranks of input tensors
assert len(data_shape) > 0, "Data must not be a scalar"
assert len(indices_shape) > 0, "Indices must not be a scalar"
assert (batch_dims + indices_shape[-1]) <= len(data_shape), \
"Length of a tuple with indices must not exceed a rank of data tensor excluding batch dimensions"
# compute output shape
if batch_dims > 0:
if is_fully_defined(data_shape[:batch_dims]):
batch = [np.prod(data_shape[:batch_dims]).tolist()]
else:
batch = [dynamic_dimension_value]
else:
batch = []
slice_shape = list(data_shape[(batch_dims + indices_shape[-1]):])
output_shape = batch + list(indices_shape[batch_dims:-1]) + slice_shape
node.out_port(0).data.set_shape(output_shape)
# compute output value if all input values are defined
if is_fully_defined(indices_value) and is_fully_defined(data_value):
output_value = np.zeros(output_shape, dtype=data_value.dtype)
if batch_dims == 0:
output_indices_range = int64_array(indices_shape[:-1])
for output_index in np.ndindex(tuple(output_indices_range)):
indices_tuple = indices_value[output_index]
output_value[output_index] = data_value[tuple(
indices_tuple.T)]
else:
batch_dims_range = int64_array(indices_shape[:batch_dims])
for batch_indices in np.ndindex(tuple(batch_dims_range)):
# compute batch index in output tensor
batch_ind = 0
num_elements = 1
for ind in reversed(range(len(batch_dims_range))):
batch_ind += batch_indices[ind] * num_elements
num_elements *= batch_dims_range[ind]
output_indices_range = int64_array(
indices_shape[batch_dims:-1])
for output_index in np.ndindex(
tuple(output_indices_range)):
tmp_ind = batch_indices + output_index
indices_tuple = tuple(indices_value[tmp_ind].T)
full_input_ind = batch_indices + indices_tuple
full_output_ind = tuple(np.array([batch_ind
]).T) + output_index
output_value[full_output_ind] = data_value[
full_input_ind]
node.out_port(0).data.set_value(output_value)
def repack_weights(graph: Graph, match: dict):
"""
Repack weights into general format (described above) and reorder gates.
"""
rnn_layer = match['rnn_layer']
W = match['W'].value.copy()
R = match['R'].value.copy()
num_directions = 2 if rnn_layer.direction == 'bidirectional' else 1
graph.remove_edge(match['W'].id, rnn_layer.id)
graph.remove_edge(match['R'].id, rnn_layer.id)
# find optional 'B' biases blob
if 3 in rnn_layer.in_nodes():
# TODO: check if 'bin': 'B' attribute is assigned to this edge
B = rnn_layer.in_node(3).value.copy()
graph.remove_edge(rnn_layer.in_node(3).id, rnn_layer.id)
else:
B_shape = [
num_directions,
2 * rnn_layer.multiplier * rnn_layer.hidden_size
] # from ONNX spec
B = np.full(B_shape, 0, dtype=np.float32)
# Add extra dimensions for W, R and B for easier repacking and reordering
B = B.reshape([
num_directions, # 0: num of directions
rnn_layer.num_layers, # 1: num_layers
2, # 2: two input parts of the matrix: W, R
rnn_layer.
multiplier, # 3: four output parts of the matrix for all gates in order: i, o, f, c
rnn_layer.hidden_size, # 4: output size per direction and gate
])
W, R = [
x.reshape([
num_directions, # 0: num of directions
rnn_layer.num_layers, # 1: num_layers
rnn_layer.
multiplier, # 2: four output parts of the matrix for all gates in order: i, o, f, c
rnn_layer.hidden_size, # 3: output size per direction and gate
-1
]) # 4: input size/hidden size in W/R correspondingly
for x in (W, R)
]
input_size = match['input'].shape[2]
assert compatible_dims(input_size, W.shape[-1])
# Reorder gates: iofc --> fico
gate_reorder = rnn_layer.gate_order
W, R = (np.take(x, gate_reorder, axis=2) for x in (W, R))
B = np.take(B, gate_reorder, axis=3)
for blob, port in [(W, 1), (R, 2), (B, 3)]:
Op.create_and_connect_input_data_node(
graph, rnn_layer, {
'value': blob,
'shape': np.array(blob.shape, dtype=np.int64)
}, {
'in': port,
'permutation': None
})