def load_components(file_descr, graph, component_layer_map=None):
num_components = collect_until_token_and_read(file_descr, b'<NumComponents>')
log.debug('Network contains {} components'.format(num_components))
is_nnet3 = False if component_layer_map is None else True
if not is_nnet3:
collect_until_token(file_descr, b'<Components>')
all_components = list()
name = ""
for _ in range(num_components):
if is_nnet3:
name = collect_until_token_and_read(file_descr, b'<ComponentName>', np.string_)
component_type = find_next_component(file_descr)
if component_type == end_of_nnet_tag.lower()[1:-1]:
break
start_index = file_descr.tell()
end_tag, end_index = find_end_of_component(file_descr, component_type)
# read dim info where possible to simplify shape calculation for MemoryOffset
# shape calculation for MemoryOffset can't be done through shape of previous layer because
# it is separated in 2 parts to remove cycle from graph
file_descr.seek(start_index)
dim = 0
try:
collect_until_token(file_descr, b'<Dim>', size_search_zone=end_index - start_index)
cur_index = file_descr.tell()
if start_index < cur_index < end_index:
dim = read_binary_integer32_token(file_descr)
else:
file_descr.seek(start_index)
except Error:
file_descr.seek(start_index)
if is_nnet3:
if name in component_layer_map:
layer_id = component_layer_map[name][0]
for layer in component_layer_map[name]:
node = Node(graph, layer)
node['parameters'] = get_parameters(file_descr, start_index, end_index)
node['op'] = component_type
# Read dim info where possible to simplify shape calculation for MemoryOffset
for o_n_name, params in node.get_outputs():
o_n = Node(graph, o_n_name)
if o_n['op'] == 'MemoryOffset' and dim != 0:
o_n['parameters']['element_size'] = dim
else:
raise Error("Something wrong with layer {}".format(name))
else:
layer_id = graph.unique_id(prefix=component_type)
graph.add_node(layer_id,
parameters=get_parameters(file_descr, start_index, end_index),
op=component_type,
kind='op')
all_components.append(layer_id)
log.debug('{} (type is {}) was loaded'.format(layer_id, component_type))
return all_components
def load_kalid_nnet1_model(graph, file_descr, name):
prev_layer_id = 'Parameter'
graph.add_node(prev_layer_id,
name=prev_layer_id,
kind='op',
op='Parameter',
parameters=None)
used_layers = set()
while True:
component_type = find_next_component(file_descr)
if component_type == end_of_nnet_tag.lower()[1:-1]:
break
layer_o = read_binary_integer32_token(file_descr)
layer_i = read_binary_integer32_token(file_descr)
if component_type == 'parallelcomponent':
prev_layer_id = load_parallel_component(file_descr, graph,
prev_layer_id)
find_end_of_component(file_descr, component_type)
continue
start_index = file_descr.tell()
end_tag, end_index = find_end_of_component(file_descr, component_type)
end_index -= len(end_tag)
layer_id = graph.unique_id(prefix=component_type)
graph.add_node(layer_id,
parameters=get_parameters(file_descr, start_index,
end_index),
op=component_type,
kind='op',
layer_i=layer_i,
layer_o=layer_o)
prev_node = Node(graph, prev_layer_id)
if prev_node.op == 'Parameter':
prev_node['shape'] = np.array([1, layer_i], dtype=np.int64)
prev_node.add_output_port(0)
Node(graph, layer_id).add_input_port(0)
graph.create_edge(
prev_node, Node(graph, layer_id), 0, 0,
create_edge_attrs(prev_layer_id, layer_id, prev_layer_id))
used_layers.add(prev_layer_id)
prev_layer_id = layer_id
log.debug('{} (type is {}) was loaded'.format(prev_layer_id,
component_type))
# Tensor names information corresponding to a node is stored on outgoing edges.
# As output nodes do not have outgoing edges, fake outputs are required. In the following code
# for each output Identity node is added, and tensor name for the output is kept
# on (output, fake output) edge. After Result nodes adding transformation fake outputs
# are deleted from graph.
output_layers = graph.nodes - used_layers
add_outputs_identity(
graph, output_layers, lambda g, output, fake_output: g.create_edge(
Node(g, output), Node(g, fake_output), 0, 0,
create_edge_attrs(output, fake_output, output)))
def load_kalid_nnet1_model(file_descr, name):
graph = Graph(name=name)
prev_layer_id = 'Input'
graph.add_node(prev_layer_id,
name=prev_layer_id,
kind='op',
op='Input',
parameters=None)
input_shape = []
while True:
component_type = find_next_component(file_descr)
if component_type == end_of_nnet_tag.lower()[1:-1]:
break
layer_o = read_binary_integer32_token(file_descr)
layer_i = read_binary_integer32_token(file_descr)
if component_type == 'parallelcomponent':
prev_layer_id = load_parallel_component(file_descr, graph,
prev_layer_id)
continue
start_index = file_descr.tell()
end_tag, end_index = find_end_of_component(file_descr, component_type)
end_index -= len(end_tag)
layer_id = graph.unique_id(prefix=component_type)
graph.add_node(layer_id,
parameters=get_parameters(file_descr, start_index,
end_index),
op=component_type,
kind='op',
layer_i=layer_i,
layer_o=layer_o)
prev_node = Node(graph, prev_layer_id)
if prev_node.op == 'Input':
prev_node['shape'] = np.array([1, layer_i], dtype=np.int64)
input_shape = np.array([1, layer_i], dtype=np.int64)
graph.add_edge(prev_layer_id, layer_id,
**create_edge_attrs(prev_layer_id, layer_id))
prev_layer_id = layer_id
log.debug('{} (type is {}) was loaded'.format(prev_layer_id,
component_type))
return graph, input_shape
def load_kalid_nnet2_model(file_descr, nnet_name):
graph = Graph(name=nnet_name)
input_name = 'Input'
input_shape = np.array([])
graph.add_node(input_name,
name=input_name,
kind='op',
op='Input',
parameters=None,
shape=None)
prev_layer_id = input_name
collect_until_token(file_descr, b'<Nnet>')
num_components = read_token_value(file_descr, b'<NumComponents>')
log.debug('Network contains {} components'.format(num_components))
collect_until_token(file_descr, b'<Components>')
for _ in range(num_components):
component_type = find_next_component(file_descr)
if component_type == end_of_nnet_tag.lower()[1:-1]:
break
start_index = file_descr.tell()
end_tag, end_index = find_end_of_component(file_descr, component_type)
layer_id = graph.unique_id(prefix=component_type)
graph.add_node(layer_id,
parameters=get_parameters(file_descr, start_index,
end_index),
op=component_type,
kind='op')
prev_node = Node(graph, prev_layer_id)
if prev_node.op == 'Input':
parameters = Node(graph, layer_id).parameters
input_dim = read_token_value(parameters, b'<InputDim>')
prev_node['shape'] = np.array([1, input_dim], dtype=np.int64)
input_shape = np.array([1, input_dim], dtype=np.int64)
graph.add_edge(prev_layer_id, layer_id,
**create_edge_attrs(prev_layer_id, layer_id))
prev_layer_id = layer_id
log.debug('{} (type is {}) was loaded'.format(prev_layer_id,
component_type))
return graph, input_shape
def test_find_next_component_eoc(self):
component = b'<LstmProjectedStreams>'
test_file = b'<!EndOfComponent>' + component + b'<tag><!EndOfComponent></Nnet>'
self.assertEqual(find_next_component(self.bytesio_from(test_file)),
component.decode('ascii').lower()[1:-1])
def test_find_next_component_end_of_nnet(self):
test_file = b'<Nnet>somefakeinfo<another>info<tag><!EndOfComponent></Nnet>'
self.assertEqual(find_next_component(self.bytesio_from(test_file)),
end_of_nnet_tag.lower()[1:-1])
