def serialize_constants_recursively(graph: Graph, bin_file, data_type, bin_hashes):
nodes = sorted(graph.nodes())
for node in nodes:
node = Node(graph, node)
if node.kind == 'data' and node.value is not None and \
any('bin' in d for u, v, d in graph.out_edges(node.node, data=True)):
# avoid array copying while taking hash
blob = node.value if node.value.ndim > 0 else node.value.reshape((1))
assert is_fully_defined(blob), 'The constant value cannot contain dynamic values'
if isinstance(blob, np.ma.masked_array):
blob = np.ma.getdata(blob)
blob_hash = hashlib.sha512(np.ascontiguousarray(blob).view(np.uint8)).hexdigest()
if blob_hash in bin_hashes and np.array_equal(blob, bin_hashes[blob_hash]['blob']):
graph.node[node.node]['offset'] = bin_hashes[blob_hash]['offset']
graph.node[node.node]['size'] = bin_hashes[blob_hash]['size']
graph.node[node.node]['blob_precision'] = np_data_type_to_precision(blob.dtype)
update_offset_size_in_const_node(node)
else:
start = bin_file.tell()
blob.tofile(bin_file)
end = bin_file.tell()
graph.node[node.node]['offset'] = start
graph.node[node.node]['size'] = end - start
graph.node[node.node]['blob_precision'] = np_data_type_to_precision(blob.dtype)
bin_hashes[blob_hash] = {'offset': graph.node[node.node]['offset'],
'size': graph.node[node.node]['size'], 'blob': blob}
update_offset_size_in_const_node(node)
assert (blob.dtype.itemsize * np.prod(node.shape) == end - start) or \
node.has_valid('force_shape'), node.attrs()
log.debug(
"Detected binary for graph: '{}', node: '{}', id: {}, shape: '{}', offset: '{}', size: '{}'".format(
graph, node.soft_get('name'), node.id, node.shape, node.offset, node.size))
# separate loop for sub-graph to dump them after all blobs for more natural blob offset ordering
# TODO: implement strict order for all blobs in entier IR
for node in nodes:
node = Node(graph, node)
# Dump blobs recursively if sub-graphs are present in the node
if node.has_valid('sub_graphs'):
for sub_graph_attr_name in node.sub_graphs:
sub_graph = node[sub_graph_attr_name]
serialize_constants_recursively(sub_graph, bin_file, data_type, bin_hashes)
def find_and_replace_pattern(self, graph: Graph):
fifo_qd_shapes = defaultdict(list)
for node in graph.get_op_nodes():
if node.op not in ["QueueDequeue", "QueueDequeueV2"]:
continue
new_inputs = ""
fifo_qd_name = node.soft_get('name', node.id)
for port_idx, port in node.out_ports().items():
if port.disconnected():
continue
if not np_data_type_to_precision(
node.types[port_idx]) in SUPPORTED_DATA_TYPES:
raise Error("Data type {} is not supported for the"
"node {}".format(node.types[port_idx],
fifo_qd_name))
fifo_qd_shapes[fifo_qd_name].append(
dict(shape=node.shapes[port_idx],
out=port_idx,
data_type=node.types[port_idx]))
new_inputs += "{}:{}, ".format(fifo_qd_name, port_idx)
log.error(
"Found TF {} operation in the model. "
"PLEASE NOTE, the model will contain new input(s) ".format(
node.op) + new_inputs +
"created due to automatically triggered pruning transformation for this operation.",
extra={'is_warning': True})
add_input_ops(graph, fifo_qd_shapes, True)
def xml_ports(node: Node, element: Element, edges: Element):
# input ports
inputs = None # will create input section only if at least one input is available
for u, d in node.get_sorted_inputs():
if 'bin' not in d and ('xml_skip' not in d or not d['xml_skip']):
if inputs is None:
inputs = SubElement(element, 'input')
port = SubElement(inputs, 'port')
port.set('id', str(d['in']))
assert node.graph.node[u][
'shape'] is not None, 'Input shape is not calculated properly for node {}'.format(
node.id)
xml_shape(node.graph.node[u]['shape'], port)
# u is a data node that has a single producer, let's find it
assert (node.graph.node[u]['kind'] == 'data')
in_nodes = list(node.graph.in_edges(u, data=True))
assert (len(in_nodes) <= 1)
if len(in_nodes) == 1:
src, _, out_attrs = in_nodes[0]
edge = SubElement(edges, 'edge')
edge.set('from-layer', str(src))
edge.set('from-port', str(out_attrs['out']))
edge.set('to-layer', str(node.node))
edge.set('to-port', str(d['in']))
# port.set('precision', np_data_type_to_precision(node['_in_port_precision'][d['in']]))
# output ports
outputs = None
for v, d in node.get_sorted_outputs():
if 'xml_skip' not in d or not d['xml_skip']:
if outputs is None:
outputs = SubElement(element, 'output')
port = SubElement(outputs, 'port')
port.set('id', str(d['out']))
# we need to check operation type, if it is const op, we don't renumber out ports
# because they are already counted from zero
port_id = d['out'] - len(
node.in_nodes()) if node.type != 'Const' else d['out']
data_type = node.out_port(port_id).get_data_type()
assert data_type is not None, 'The precision is not defined for the output port {} of node {}' \
''.format(port_id, node.soft_get('name'))
port.set(
'precision',
node.soft_get('force_type',
np_data_type_to_precision(data_type)))
assert node.graph.node[v]['shape'] is not None, 'Output shape is not calculated properly for node {}' \
''.format(node.id)
tensor_names = node.out_port(port_id).get_tensor_names(
port_renumber=True)
if tensor_names:
port.set('names', ','.join(tensor_names))
xml_shape(node.graph.node[v]['shape'], port)
def find_and_replace_pattern(self, graph: Graph):
iter_get_next_shapes = defaultdict(list)
for iter_get_next in graph.get_op_nodes(op='IteratorGetNext'):
iter_get_next_name = iter_get_next.soft_get(
'name', iter_get_next.id)
for port in iter_get_next.out_ports():
if not np_data_type_to_precision(
iter_get_next.types[port]) in SUPPORTED_DATA_TYPES:
raise Error(
"In IteratorGetNext node '{}' data type '{}' is not supported"
.format(iter_get_next_name, iter_get_next.types[port]))
iter_get_next_shapes[iter_get_next_name].append(
dict(shape=iter_get_next.shapes[port],
out=port,
data_type=iter_get_next.types[port]))
add_input_ops(graph, iter_get_next_shapes, True)
def prepare_obj_for_dump(obj: object):
if isinstance(obj, dict):
return {k: prepare_obj_for_dump(v) for k, v in obj.items()}
elif isinstance(obj, np.ndarray):
if obj.ndim == 0:
return obj.item()
else:
return [prepare_obj_for_dump(elem) for elem in obj]
elif isinstance(obj, list):
return [prepare_obj_for_dump(elem) for elem in obj]
elif isinstance(obj, type):
try:
return np_data_type_to_precision(obj)
except:
log.error('Unsupported data type: {}'.format(str(obj)))
return str(obj)
elif isinstance(obj, np.generic):
return obj.item()
else:
return str(obj)
def custom_type_casting_and_packing(node: Node, value, dst_type):
"""
Custom types are not supported by numpy but we still need to write it to the .bin file in a compact way.
To do so we prepare bit representation of int4/uint4 values and store them in a numpy friendly data type.
We pack int4/uint4 values into uint8 type (two int4/uint4 numbers fit in uint8).
If the number of elements in the blob is odd we pad them with zero value to be able to fit the bit sequence
into the uint8 array.
Example: we need to represent 5 elements of int4 dtype
we would pad them to 6 element with the last element as zero and we would pack them into 3 uint8 values
"""
assert dst_type in [packed_U4, packed_I4]
minimum_regular_dtype = np.uint8 if dst_type == packed_U4 else np.int8
# initial casing from the source type to the numpy-friendly type which could absorb all the values of dst_type
casted_to_regular_type = Cast.helper_value_propagation(
node.soft_get('name', node.id), value, minimum_regular_dtype)
# packing the values
data_shape = node.out_port(0).data.get_shape()
assert data_shape is not None
data_size = np.prod(data_shape)
num_bits = 4
assert num_bits < 8 and 8 % num_bits == 0, "Packing algorithm for the data types stored in 1, 2 or 4 bits"
num_values_fitting_into_uint8 = 8 // num_bits
pad = (-data_size) % num_values_fitting_into_uint8
flattened = casted_to_regular_type.flatten()
padded = np.concatenate(
(flattened, np.zeros([pad], dtype=minimum_regular_dtype)))
assert np.prod(padded.shape) % num_values_fitting_into_uint8 == 0
bit_order_little = (padded[:, None] &
(1 << np.arange(num_bits)) > 0).astype(np.uint8)
bit_order_big = np.flip(bit_order_little, axis=1)
bit_order_big_flattened = bit_order_big.flatten()
packed = np.packbits(bit_order_big_flattened)
node.out_node(0)['force_shape'] = data_shape.copy()
node.out_node(0)['force_type'] = np_data_type_to_precision(dst_type)
node.out_port(0).data.set_value(packed)
