def replace_pattern(self, graph: Graph, match: dict):
"""
Adds layers with type 'Const' that produce blob from 'bin' file. The pass finds data nodes with one output which
doesn't have edge with 'bin' attribute (or with two outputs and at least one output havent 'bin' attr)
and generate Const op node before the node and data node before the Const node. The data node before 'Const'
node is needed because the op node dumps input tensors to bin file.
"""
node = match['data']
if len(node.in_nodes()) > 0:
return
if self._check_bin_attrs(node):
if node.has_valid('value'):
const_node_name = graph.unique_id(node.id + '_const')
log.debug("Added Const node '{}'".format(const_node_name))
const_node = Const(
graph, {
'name':
const_node_name,
'value':
node.value,
'force_shape':
node.soft_get('force_shape', None),
'override_output_shape':
node.has_valid('force_shape'),
'force_type':
node.soft_get('force_type', None),
'correct_data_type':
node.soft_get('correct_data_type', None),
}).create_node()
const_node.add_input_port(0)
graph.add_edges_from([(const_node_name, node.id, {'out': 0})])
node_copy = node.copy_node()
const_node.type_infer(const_node)
graph.add_edges_from([(node_copy.id, const_node_name, {
'in': 0,
'bin': 'custom'
})])
elif not self._check_that_node_from_body(node):
log.debug('node = {}'.format(node.graph.node[node.id]))
raise Error(
'Discovered data node without inputs and value, node.name = {}, consumer.name = {}. '
+ refer_to_faq_msg(23), node.soft_get('name'),
node.out_node().soft_get('name')
if len(node.out_nodes()) else "<no consumer>")
def extract(cls, node: Node):
unsupported_attrs = []
for attr_name in [
'adjoint_a', 'adjoint_b', 'a_is_sparse', 'b_is_sparse'
]:
if attr_name in node.pb.attr and node.pb.attr[attr_name].b:
unsupported_attrs.append(attr_name)
if len(unsupported_attrs) != 0:
raise Error('MatMul operation {} use unsupported attrs: {}'.format(
node.id, unsupported_attrs))
MatMul.update_node_stat(
node, {
'transpose_a': node.pb.attr['transpose_a'].b,
'transpose_b': node.pb.attr['transpose_b'].b,
})
return cls.enabled
def convert_blob(blob: np.ndarray, dst_type: type):
if blob.dtype == dst_type:
return blob, None, None
converted_blob = blob.astype(dtype=dst_type, casting="unsafe")
if dst_type in (np.int32, np.int64, np.uint8,
np.int8) and not np.array_equal(blob, converted_blob):
raise Error(
'The conversion of blob with value "{}" to dst_type "{}" results in rounding'
.format(blob, dst_type))
finite_match = (np.isfinite(blob) != np.isfinite(converted_blob))
zero_match = ((blob == 0) != (converted_blob == 0))
finite_match_count = np.count_nonzero(finite_match)
zero_match_count = np.count_nonzero(zero_match)
return converted_blob, finite_match_count, zero_match_count
def __init__(self, graph: Graph, attrs: dict):
mandatory_props = {
'type': __class__.op,
'op': __class__.op,
'levels': None,
'is_eltwise': True,
# flag to switch between dumping FakeQuantize as statistics and keeping it as layer in IR
'keep_in_IR': None,
'infer': __class__.infer,
'in_ports_count': 5,
'out_ports_count': 1,
}
super().__init__(graph, mandatory_props, attrs)
if self.attrs['levels'] is None:
raise Error("FakeQuantize operation has no levels parameter")
# TODO remove following lines after FakeQuantize supported for int8 workflow
self.attrs['keep_in_IR'] = self.attrs['levels'] == 2 or graph.graph['cmd_params'].keep_quantize_ops_in_IR
def _check_unique_ids():
"""
Check that idxs is unique for all registered replacements.
"""
unique_idxs = set()
for class_type, classes_set in _registered_classes_dict.items():
for cls in classes_set:
replacers = [c for c in cls.registered_cls if not hasattr(c, 'op')] + \
[c for op, c in cls.registered_ops.items() if c]
for replacer_cls in replacers:
if hasattr(replacer_cls, 'id'):
id_cls = getattr(replacer_cls, 'id')
if id_cls in unique_idxs:
raise Error('Found replacer {} with not unique id!'.format(replacer_cls))
unique_idxs.add(id_cls)
log.debug("All replacers has unique idxs.")
def collect_until_token(file_desc: io.BufferedReader, token):
"""
Read from file until the token
:param file_desc: file descriptor
:return:
"""
while True:
# usually there is the following structure <CellDim> DIM<ClipGradient> VALUEFM
res = collect_until_whitespace(file_desc)
if res == token or res[-len(token):] == token:
return
if isinstance(file_desc, io.BytesIO):
size = len(file_desc.getbuffer())
elif isinstance(file_desc, io.BufferedReader):
size = os.fstat(file_desc.fileno()).st_size
if file_desc.tell() == size:
raise Error('End of the file. Token {} not found. {}'.format(token, file_desc.tell()))
def convert(graph: nx.MultiDiGraph, data_type_str: str):
for node_name, node_attrs in graph.nodes(data=True):
node = Node(graph, node_name)
# if the data type is forcibly set then use it
if node.has_valid('force_precision'):
real_data_type_str = node_attrs['force_precision']
else:
real_data_type_str = data_type_str
node_attrs['precision'] = data_type_str_to_precision(
real_data_type_str)
if node.kind == 'data' and node.value is not None:
try:
convert_blob(graph, node,
data_type_str_to_np(real_data_type_str))
except Exception as e:
raise Error('Coudn\'t convert blob {}, details: {}',
node.soft_get('name'), e) from e
def merge_nodes_permutations(graph: Graph):
# Iterate over all data nodes and check all permutations for similarity
# In case of equal permutations, this permutation will be set as attribute for data node
# otherwise exception will be raised
for node in graph.nodes():
node = Node(graph, node)
if node.kind != 'data':
continue
permutations = []
# Get all permutations from in edges
for in_node in node.in_nodes():
edge_attrs = node.graph.get_edge_data(in_node.id, node.id)[0]
if 'permutation' in edge_attrs:
permutations.append(edge_attrs['permutation'])
# Get all permutations from out edges
for out_node in node.out_nodes():
edge_attrs = node.graph.get_edge_data(node.id, out_node.id)[0]
if 'permutation' in edge_attrs:
permutations.append(edge_attrs['permutation'])
# Check that all permutations are equal
final_permutations = []
for p in permutations:
if p is not None:
final_permutations.append(p.perm)
else:
final_permutations.append(
int64_array(np.arange(node.shape.size)))
if len(final_permutations) == 0:
continue
if not all([
np.array_equal(final_permutations[0], perm)
for perm in final_permutations
]):
raise Error(
'Permutations requested for {} data node are not equal! List of permutations: {}'
''.format(node.name, [p.perm for p in permutations]))
assert not node.has_valid('permutation') or np.array_equal(
node.permutation, permutations[0])
node['permutation'] = permutations[0]
def extract(cls, node):
attrs = get_mxnet_layer_attrs(node.symbol_dict)
act_type = attrs.str('act_type', 'relu')
if act_type == 'sigmoid':
act_class = Sigmoid
elif act_type == 'tanh':
act_class = Tanh
elif act_type == 'relu':
act_class = ReLU
elif act_type == 'softrelu':
act_class = SoftPlus
else:
raise Error(
"Operation '{}' not supported. Please register it as custom op. "
+ refer_to_faq_msg(86), act_type)
act_class.update_node_stat(node)
return cls.enabled
def decode_name_with_port(input_model: InputModel, node_name: str):
"""
Decode name with optional port specification w/o traversing all the nodes in the graph
TODO: in future node_name can specify input/output port groups and indices (58562)
:param input_model: Input Model
:param node_name: user provided node name
:return: decoded place in the graph
"""
# Check exact match with one of the names in the graph first
node = input_model.get_place_by_tensor_name(node_name)
if node:
return node
# TODO: Add support for input/output group name and port index here (58562)
# Legacy frontends use format "number:name:number" to specify input and output port indices
# For new frontends this logic shall be extended to additionally support input and output group names
raise Error('There is no node with name {}'.format(node_name))
def check_shapes_consistency(self):
data_nodes = self.get_data_nodes()
data_nodes_with_wrong_shapes = []
for data_node in data_nodes:
if not data_node.has('shape'):
data_nodes_with_wrong_shapes.append(
(data_node.name, "no shape attribute"))
continue
if data_node.shape is not None and not isinstance(
data_node.shape, np.ndarray):
data_nodes_with_wrong_shapes.append(
(data_node.name, type(data_node.shape)))
if len(data_nodes_with_wrong_shapes) > 0:
raise Error(
"Graph contains data nodes ({}) with inconsistent shapes: {}".
format(len(data_nodes_with_wrong_shapes),
data_nodes_with_wrong_shapes))
def onnx_attr(node: Node, name: str, field: str, default=None, dst_type=None):
""" Retrieves ONNX attribute with name `name` from ONNX protobuf `node.pb`.
The final value is casted to dst_type if attribute really exists.
The function returns `default` otherwise.
"""
attrs = [a for a in node.pb.attribute if a.name == name]
if len(attrs) == 0:
# there is no requested attribute in the protobuf message
return default
elif len(attrs) > 1:
raise Error('Found multiple entries for attribute name {} when at most one is expected. Protobuf message with '
'the issue: {}.', name, node.pb)
else:
res = getattr(attrs[0], field)
if dst_type is not None:
return dst_type(res)
else:
return res
def shape_inference(graph: Graph):
nodes = pseudo_topological_sort(graph)
for node in nodes:
node = Node(graph, node)
if node.has_and_set('need_shape_inference'):
old_out_shapes = [
port.data.get_shape() for port in node.out_ports().values()
]
node.infer(node)
new_out_shapes = [
port.data.get_shape() for port in node.out_ports().values()
]
for shape1, shape2 in zip(old_out_shapes, new_out_shapes):
if shape1 is not None and not np.array_equal(shape1, shape2):
raise Error(
"After partial shape inference were found shape collision for node {} (old shape: {}, new shape: {})"
.format(node.name, shape1, shape2))
node.need_shape_inference = False
def replace_sub_graph(self, graph: Graph, match: [dict, SubgraphMatch]):
node = match['reduce']
connected_in_ports = [
port for port in node.in_ports().values()
if not port.disconnected()
]
if len(connected_in_ports) == 1:
# if the 'axis' is None then we still add a second input to the layer with a 1D array with 1 element equal
# to None. The infer function handles this case because the input shape is known at this stage only
if node.has('axis'):
const = Const(graph, {'value': node.axis}).create_node()
node.add_input_port(1, skip_if_exist=True)
const.out_port(0).connect(node.in_port(1))
del graph.node[node.id]['axis']
else:
raise Error(
'Can not deduce `reduce_axis` for {}: only one in_port and no `axis` parameter.'
''.format(node.op))
def find_and_replace_pattern(self, graph: Graph):
for node in graph.get_op_nodes():
node_type = node.soft_get('type').lower()
name = node.soft_get('name', node.id)
if node.soft_get('version', None) == 'opset1' and node_type not in self.opset_1_types:
raise Error('Node {} has `version` attribute set to `opset1`, but it is a reserved word, '
'please use another'.format(name))
if not node.has_valid('version'):
if node_type in self.opset_1_types:
node['version'] = 'opset1'
elif node_type in self.opset_1_experimental_ops:
node['version'] = 'experimental'
else:
node['version'] = 'extension'
log.error('Please set `version` attribute for node {} with type={}'
''.format(name, node.soft_get('type')), extra={'is_warning': True})
def replace_sub_graph(self, graph: Graph, match: [dict, SubgraphMatch]):
node = match['transpose']
connected_in_ports = [
port for port in node.in_ports().values()
if not port.disconnected()
]
if len(connected_in_ports) == 1:
if node.has_valid('order'):
const = Const(graph, {'value': node.order}).create_node()
node.add_input_port(1, skip_if_exist=True)
const.out_port(0).connect(node.in_port(1))
del graph.node[node.id]['order']
elif node.has('order') and node.order is None:
assert node.has_and_set('reverse_order')
else:
raise Error(
'Can not deduce transpose `order` for {}: only one in_port and no `order` parameter.'
''.format(node.op))
def find_and_replace_pattern(self, graph: Graph):
for squeeze_node in graph.get_op_nodes(op='Squeeze'):
if len(squeeze_node.in_nodes()) == 1 and squeeze_node.has_valid(
'squeeze_dims'):
dims_node = Const(
graph, {
'name': squeeze_node.id + '/Dims',
'value': int64_array(squeeze_node.squeeze_dims)
}).create_node()
squeeze_node.in_port(1).connect(dims_node.out_port(0))
del squeeze_node['squeeze_dims']
elif len(squeeze_node.in_nodes()) == 2:
log.debug('The Squeeze node "{}" is already normalized'.format(
squeeze_node.name))
else:
raise Error(
'The Squeeze layer "{}" should either have 2 inputs or one input and an "squeeze_dims" '
'attribute'.format(squeeze_node.soft_get('name')))
def extract(cls, node):
if get_onnx_opset_version(node) < 10:
starts = int64_array(onnx_attr(node, 'starts', 'ints', default=[]))
ends = int64_array(onnx_attr(node, 'ends', 'ints', default=[]))
axes = int64_array(onnx_attr(node, 'axes', 'ints', default=[]))
if len(starts) == 0 or len(ends) == 0:
raise Error(
"starts or/and ends are not specified for the node {}".
format(node.name))
if len(axes) == 0:
axes = np.arange(len(starts), dtype=np.int)
attrs = {'axes': axes, 'starts': starts, 'ends': ends}
AttributedSlice.update_node_stat(node, attrs)
else: # onnx_opset_version >= 10
Slice.update_node_stat(node)
return cls.enabled
def backend_attrs(self):
version = self.get_opset()
if version == 'opset2':
return [
'eps',
('across_channels',
lambda node: bool_to_str(node, 'across_channels')),
('normalize_variance',
lambda node: bool_to_str(node, 'normalize_variance'))
]
elif version == 'opset6':
return [
'eps', 'eps_mode',
('normalize_variance',
lambda node: bool_to_str(node, 'normalize_variance'))
]
else:
raise Error('Unsupported MVN opset version "{}"'.format(version))
def extract_port(node_port):
if isinstance(node_port, tuple):
node = node_port[0]
port = node_port[1]
else:
node = node_port
port = 0
# 'data' nodes do not have 'out' edge attribute but always has one output
out_ids = [
attr['out']
for _, __, attr in node.graph.out_edges(node.id, data=True)
if 'out' in attr
]
if len(set(out_ids)) > 1 and not isinstance(node_port, tuple):
raise Error(
'Node {} has more than one outputs. Provide output port explicitly. '
.format(node.name))
return node, port
def find_and_replace_pattern(self, graph: Graph):
iter_get_next_shapes = defaultdict(list)
for iter_get_next in graph.get_op_nodes(op='IteratorGetNext'):
iter_get_next_name = iter_get_next.soft_get(
'name', iter_get_next.id)
for port in iter_get_next.out_ports():
if not np_data_type_to_precision(
iter_get_next.types[port]) in SUPPORTED_DATA_TYPES:
raise Error(
"In IteratorGetNext node '{}' data type '{}' is not supported"
.format(iter_get_next_name, iter_get_next.types[port]))
iter_get_next_shapes[iter_get_next_name].append(
dict(shape=iter_get_next.shapes[port],
out=port,
data_type=iter_get_next.types[port]))
add_input_ops(graph, iter_get_next_shapes, True)
def tf_tensor_content(tf_dtype, shape, pb_tensor):
type_helper = tf_data_type_decode[tf_dtype] if tf_dtype in tf_data_type_decode else None
if type_helper is None:
raise Error("Data type is unsupported: {}. " +
refer_to_faq_msg(50), tf_dtype)
if pb_tensor.tensor_content:
value = np.array(np.frombuffer(pb_tensor.tensor_content, type_helper[0]))
else:
# load typed value
if type_helper[0] != np.str:
value = np.array(type_helper[1](pb_tensor), dtype=type_helper[0])
else:
try:
value = np.array(type_helper[1](pb_tensor), dtype=type_helper[0])
except UnicodeDecodeError:
log.error(
'Failed to parse a tensor with Unicode characters. Note that Inference Engine does not support '
'string literals, so the string constant should be eliminated from the graph.',
extra={'is_warning': True})
value = np.array(type_helper[1](pb_tensor))
if len(shape) == 0 or shape.prod() == 0:
if len(value) == 1:
# return scalar if shape is [] otherwise broadcast according to shape
return np.array(value[0], dtype=type_helper[0])
else:
# no shape, return value as is
return value
if len(value) != shape.prod():
log.warning("Shape and content size of tensor don't match, shape: {} content size: {}".
format(shape, len(value)))
# broadcast semantics according to TensorFlow v1.5 documentation:
# The argument value can be a constant value, or a list of values of type dtype. If value is a list,
# then the length of the list must be less than or equal to the number of elements implied by the shape
# argument (if specified). In the case where the list length is less than the number of elements specified
# by shape, the last element in the list will be used to fill the remaining entries.
value_flatten = value.flatten()
add_value = value_flatten[-1]
add_length = shape.prod() - len(value_flatten)
value = np.concatenate([value_flatten, np.full([add_length], add_value)])
return value.reshape(shape)
def find_and_replace_pattern(self, graph: Graph):
for node in graph.get_op_nodes(squeeze_axis=True):
name = node.soft_get('name', node.id)
for out_port in node.out_ports().values():
if node.has_valid('axis'):
squeeze_node = create_op_with_const_inputs(
graph, Squeeze, {1: np.array(node.axis)},
{'name': name + '/Squeeze_'})
out_port.get_connection().insert_node(squeeze_node)
elif node.is_in_port_connected(1):
squeeze_node = Squeeze(graph, {
'name': name + '/Squeeze_'
}).create_node()
out_port.get_connection().insert_node(squeeze_node)
node.in_port(1).get_connection().add_destination(
squeeze_node.in_port(1))
else:
raise Error(
'Unknown axis to squeeze for node {}'.format(name))
def shape_insert(shape: [np.ndarray, list], pos: int,
obj: [int, list, np.ndarray, dynamic_dimension]):
"""
Insert element(s) in the input tensor shape (presumably the numpy masked array) specified by position pos.
The function is implemented to avoid usage of np.insert which corrupts information about the masked elements.
:param shape: the shape object to insert element(s) to
:param pos: the position to insert the elements into
:param obj: the list or a single integer or the dynamic_dimension_value or numpy array to insert
:return: shape with inserted elements
"""
if isinstance(obj,
(int, np.int64, np.int32)) or obj is dynamic_dimension_value:
return shape_insert(shape, pos, [obj])
elif isinstance(obj, (np.ndarray, list)):
return np.ma.concatenate((shape_array(shape[:pos]), shape_array(obj),
shape_array(shape[pos:])))
else:
raise Error('Incorrect parameter type of "obj": {}'.format(type(obj)))
def extract(cls, node):
pb = node.parameters
collect_until_token(pb, b'<LinearParams>')
weights, weights_shape = read_binary_matrix(pb)
tag = find_next_tag(pb)
read_placeholder(pb, 1)
if tag != '<BiasParams>':
raise Error('FixedAffineComponent must contain BiasParams')
biases = read_binary_vector(pb)
mapping_rule = {
'out-size': weights_shape[0],
'transpose_weights': True,
}
embed_input(mapping_rule, 1, 'weights', weights)
embed_input(mapping_rule, 2, 'biases', biases)
FullyConnected.update_node_stat(node, mapping_rule)
return cls.enabled
def _set_value(self, value):
if self.node.graph.stage == 'front':
raise Error("set_value is not applicable for graph front phase")
else:
data_node_caller = self.node.in_node if self.type == 'in' else self.node.out_node
data_node = data_node_caller(self.idx, control_flow=self.control_flow)
const_node = data_node.in_node(control_flow=self.control_flow) if self.type == 'in' else self.node
force_shape = data_node.soft_get('force_shape', const_node.soft_get('force_shape', None))
shape = int64_array(value.shape if force_shape is None else force_shape)
# Set value to data node
data_node.value = value
data_node.shape = shape
# Set value to constant producer
if const_node.soft_get('type') == 'Const':
const_node.value = value
const_node.shape = shape
def save_custom_replacement_config_file(descriptions: list,
file_name: str):
"""
Save custom layer(s) description(s) to the file.
:param file_name: file to save description information to.
:param descriptions: list with instances of the CustomLayerDescriptor classes.
:return: True if operation is successful.
"""
try:
json.dump([
replacement_desc.get_config_file_representation()
for replacement_desc in descriptions
],
open(file_name, "w"),
indent=4,
sort_keys=True)
except Exception as ex:
raise Error("failed to update configuration file {}: {}".format(
file_name, str(ex)))
def find_and_replace_pattern(self, graph: Graph):
unsupported_nodes = []
for node in graph.nodes():
node = Node(graph, node)
if node.kind == 'op' and node.soft_get('type') in self.unsupported_operations:
input_shape = node.in_node(0).shape
if len(input_shape) > 4:
unsupported_nodes.append((node.id, node.type))
if len(unsupported_nodes) == 0:
return
error_message = "\nOperations below were marked as unsupported due to they expect more than two spatial dims" \
" (input shape length more than 4)\n"
error_message += "List of unsupported operations ({})\n".format(len(unsupported_nodes))
for node, type in unsupported_nodes:
error_message += " {} {}\n".format(type, node)
raise Error(error_message)
def read_token_value(file_desc: io.BufferedReader, token: bytes = b'', value_type: type = np.uint32):
"""
Get value of the token.
Read next token (until whitespace) and check if next teg equals token
:param file_desc: file descriptor
:param token: token
:param value_type: type of the reading value
:return: value of the token
"""
getters = {
np.uint32: read_binary_integer32_token,
np.uint64: read_binary_integer64_token,
np.bool: read_binary_bool_token
}
current_token = collect_until_whitespace(file_desc)
if token != b'' and token != current_token:
raise Error('Can not load token {} from Kaldi model'.format(token) +
refer_to_faq_msg(94))
return getters[value_type](file_desc)
def replace_op(self, graph: Graph, node: Node):
ss_node = create_op_with_const_inputs(graph, Split, {1: int64_array(1)}, {'name': 'Split_eltwise_' + node.name,
'num_splits': node['num_inputs']})
inp = node.get_inputs()
in_node = inp[0][0]
edge_attrs = inp[0][1]
graph.add_edge(in_node, ss_node.id, **edge_attrs)
if ss_node.num_splits == 2:
eltwise_node = Eltwise(graph, attrs={'name': 'Eltwise_' + node.name,
'operation': node['operation']}).create_node()
elif ss_node.num_splits > 2:
eltwise_node = EltwiseN(graph, attrs={'name': 'Eltwise_' + node.name,
'operation': node['operation']}).create_node()
else:
raise Error('Error on replacing Kaldi eltwise')
for i in range(ss_node.num_splits):
ss_node.out_port(i).get_connection().set_destination(eltwise_node.in_port(i))
return [eltwise_node.id]
