def serialize_constants_recursively(graph: Graph, bin_file, data_type,
bin_hashes):
nodes = sorted(graph.nodes())
for node in nodes:
node = Node(graph, node)
if node.kind == 'data' and node.value is not None and any(
'bin' in d
for u, v, d in graph.out_edges(node.node, data=True)):
# avoid array copying while taking hash
blob = node.value if node.value.ndim > 0 else node.value.reshape(
(1))
blob_hash = hashlib.sha512(
np.ascontiguousarray(blob).view(np.uint8)).hexdigest()
if blob_hash in bin_hashes and np.array_equal(
blob, bin_hashes[blob_hash]['blob']):
graph.node[
node.node]['offset'] = bin_hashes[blob_hash]['offset']
graph.node[node.node]['size'] = bin_hashes[blob_hash]['size']
graph.node[
node.node]['blob_precision'] = np_data_type_to_precision(
blob.dtype)
update_offset_size_in_const_node(node)
else:
start = bin_file.tell()
blob.tofile(bin_file)
end = bin_file.tell()
graph.node[node.node]['offset'] = start
graph.node[node.node]['size'] = end - start
graph.node[
node.node]['blob_precision'] = np_data_type_to_precision(
blob.dtype)
bin_hashes[blob_hash] = {
'offset': graph.node[node.node]['offset'],
'size': graph.node[node.node]['size'],
'blob': blob
}
update_offset_size_in_const_node(node)
assert (blob.dtype.itemsize * np.prod(node.shape) == end -
start) or node.has_valid('force_shape'), node.attrs()
log.debug(
"Detected binary for graph: '{}', node: '{}', id: {}, shape: '{}', offset: '{}', size: '{}'"
.format(graph, node.soft_get('name'), node.id, node.shape,
node.offset, node.size))
# separate loop for sub-graph to dump them after all blobs for more natural blob offset ordering
# TODO: implement strict order for all blobs in entier IR
for node in nodes:
node = Node(graph, node)
# Dump blobs recursively if sub-graphs are present in the node
if node.has_valid('sub_graphs'):
for sub_graph_attr_name in node.sub_graphs:
sub_graph = node[sub_graph_attr_name]
serialize_constants_recursively(sub_graph, bin_file, data_type,
bin_hashes)
def backend_attrs(self):
if self.ir_version == 10:
return [
('destination_type',
lambda node: np_data_type_to_destination_type(node.dst_type))
]
else:
return [('precision',
lambda node: np_data_type_to_precision(node.dst_type))]
def prepare_obj_for_dump(obj: object):
if isinstance(obj, dict):
return {k: prepare_obj_for_dump(v) for k, v in obj.items()}
elif isinstance(obj, np.ndarray) or isinstance(obj, list):
return [prepare_obj_for_dump(elem) for elem in obj]
elif isinstance(obj, type):
return np_data_type_to_precision(obj)
elif isinstance(obj, np.generic):
return obj.item()
else:
return obj
def xml_ports(node: Node, element: Element, edges: Element):
# input ports
inputs = None # will create input section only if at least one input is available
for u, d in node.get_sorted_inputs():
if 'bin' not in d and ('xml_skip' not in d or not d['xml_skip']):
if inputs is None:
inputs = SubElement(element, 'input')
port = SubElement(inputs, 'port')
port.set('id', str(d['in']))
assert node.graph.node[u][
'shape'] is not None, 'Input shape is not calculated properly for node {}'.format(
node.id)
xml_shape(node.graph.node[u]['shape'], port)
# u is a data node that has a single producer, let's find it
assert (node.graph.node[u]['kind'] == 'data')
in_nodes = list(node.graph.in_edges(u, data=True))
assert (len(in_nodes) <= 1)
if len(in_nodes) == 1:
src, _, out_attrs = in_nodes[0]
edge = SubElement(edges, 'edge')
edge.set('from-layer', str(src))
edge.set('from-port', str(out_attrs['out']))
edge.set('to-layer', str(node.node))
edge.set('to-port', str(d['in']))
# port.set('precision', np_data_type_to_precision(node['_in_port_precision'][d['in']]))
# output ports
outputs = None
for v, d in node.get_sorted_outputs():
if 'xml_skip' not in d or not d['xml_skip']:
if outputs is None:
outputs = SubElement(element, 'output')
port = SubElement(outputs, 'port')
port.set('id', str(d['out']))
# we need to check operation type, if it is const op, we don't renumber out ports
# because they are already counted from zero
port_id = d['out'] - len(
node.in_nodes()) if node.type != 'Const' else d['out']
data_type = node.out_port(port_id).get_data_type()
assert data_type is not None, 'The precision is not defined for the output port {} of node {}' \
''.format(port_id, node.soft_get('name'))
port.set(
'precision',
node.soft_get('force_type',
np_data_type_to_precision(data_type)))
assert node.graph.node[v]['shape'] is not None, 'Output shape is not calculated properly for node {}' \
''.format(node.id)
tensor_names = node.out_port(port_id).get_tensor_names(
port_renumber=True)
if tensor_names is not None:
port.set('names', tensor_names)
xml_shape(node.graph.node[v]['shape'], port)
def find_and_replace_pattern(self, graph: Graph):
iter_get_next_shapes = defaultdict(list)
for iter_get_next in graph.get_op_nodes(op='IteratorGetNext'):
iter_get_next_name = iter_get_next.soft_get(
'name', iter_get_next.id)
for port in iter_get_next.out_ports():
if not np_data_type_to_precision(
iter_get_next.types[port]) in SUPPORTED_DATA_TYPES:
raise Error(
"In IteratorGetNext node '{}' data type '{}' is not supported"
.format(iter_get_next_name, iter_get_next.types[port]))
iter_get_next_shapes[iter_get_next_name].append(
dict(shape=iter_get_next.shapes[port],
out=port,
data_type=iter_get_next.types[port]))
add_input_ops(graph, iter_get_next_shapes, True)
def prepare_obj_for_dump(obj: object):
if isinstance(obj, dict):
return {k: prepare_obj_for_dump(v) for k, v in obj.items()}
elif isinstance(obj, np.ndarray):
if obj.ndim == 0:
return obj.item()
else:
return [prepare_obj_for_dump(elem) for elem in obj]
elif isinstance(obj, list):
return [prepare_obj_for_dump(elem) for elem in obj]
elif isinstance(obj, type):
try:
return np_data_type_to_precision(obj)
except:
log.error('Unsupported data type: {}'.format(str(obj)))
return str(obj)
elif isinstance(obj, np.generic):
return obj.item()
else:
return str(obj)
def custom_type_casting_and_packing(node: Node, value, dst_type):
"""
Custom types are not supported by numpy but we still need to write it to the .bin file in a compact way.
To do so we prepare bit representation of int4/uint4 values and store them in a numpy friendly data type.
We pack int4/uint4 values into uint8 type (two int4/uint4 numbers fit in uint8).
If the number of elements in the blob is odd we pad them with zero value to be able to fit the bit sequence
into the uint8 array.
Example: we need to represent 5 elements of int4 dtype
we would pad them to 6 element with the last element as zero and we would pack them into 3 uint8 values
"""
assert dst_type in [packed_U4, packed_I4]
minimum_regular_dtype = np.uint8 if dst_type == packed_U4 else np.int8
# initial casing from the source type to the numpy-friendly type which could absorb all the values of dst_type
casted_to_regular_type = Cast.helper_value_propagation(
node.soft_get('name', node.id), value, minimum_regular_dtype)
# packing the values
data_shape = node.out_port(0).data.get_shape()
assert data_shape is not None
data_size = np.prod(data_shape)
num_bits = 4
assert num_bits < 8 and 8 % num_bits == 0, "Packing algorithm for the data types stored in 1, 2 or 4 bits"
num_values_fitting_into_uint8 = 8 // num_bits
pad = (-data_size) % num_values_fitting_into_uint8
flattened = casted_to_regular_type.flatten()
padded = np.concatenate(
(flattened, np.zeros([pad], dtype=minimum_regular_dtype)))
assert np.prod(padded.shape) % num_values_fitting_into_uint8 == 0
bit_order_little = (padded[:, None] &
(1 << np.arange(num_bits)) > 0).astype(np.uint8)
bit_order_big = np.flip(bit_order_little, axis=1)
bit_order_big_flattened = bit_order_big.flatten()
packed = np.packbits(bit_order_big_flattened)
node.out_node(0)['force_shape'] = data_shape.copy()
node.out_node(0)['force_type'] = np_data_type_to_precision(dst_type)
node.out_port(0).data.set_value(packed)
def backend_attrs(self):
return [('precision', lambda node: np_data_type_to_precision(node.dst_type))]
def save_restored_graph(graph: Graph, path: str, meta_data, name=None):
"""
Function to apply all necessary transforms from back stage to prepare and save restored graph and metadata.
:param graph: Graph to save
:param path: Path to saved IR
:param meta_data: Namespace with converting parameters restored from IR
:param name: Name for saved IR
:return:
"""
if name is None:
name = graph.name
precisions = set()
for op in graph.get_op_nodes():
if op.type in ('Convolution', 'MatMul'):
if op.in_port(1).get_source().node.type == 'FakeQuantize':
data_type = op.in_port(1).get_source().node.in_port(
0).get_source().node.soft_get('data_type', None)
else:
data_type = op.in_port(1).get_source().node.soft_get(
'data_type', None)
if data_type is not None:
precisions.add(np_data_type_to_precision(data_type))
else:
log.warning(
'Cannot check data type for node {} with type {}, skip it.'
.format(op.name, op.type))
precision = 'FP16' if 'FP16' in precisions else 'FP32'
# We need to run some specific passes from MO back stage.
# After some of them we need to clean up graph!
for_graph_and_each_sub_graph_recursively(
graph,
ConvolutionWithGroupsResolver().find_and_replace_pattern)
for_graph_and_each_sub_graph_recursively(
graph,
TopKNormalizer().find_and_replace_pattern)
graph.clean_up()
for_graph_and_each_sub_graph_recursively(
graph,
StridedSliceMasksNormalizer().find_and_replace_pattern)
for_graph_and_each_sub_graph_recursively(
graph,
BlobNormalizer().find_and_replace_pattern)
for_graph_and_each_sub_graph_recursively(
graph,
ConvolutionNormalizer().find_and_replace_pattern)
for_graph_and_each_sub_graph_recursively(
graph,
RemoveConstOps().find_and_replace_pattern)
for_graph_and_each_sub_graph_recursively(
graph,
CreateConstNodesReplacement().find_and_replace_pattern)
prepare_emit_ir(graph, precision, path, name, meta_info=meta_data)
