def extract(cls, node):
# borders: leftBorder, topBorder, rightBorder, bottomBordes
borders = onnx_attr(node,
'border',
'ints',
default=None,
dst_type=int64_array)
scale = onnx_attr(node,
'scale',
'ints',
default=None,
dst_type=int64_array)
# Crop reference: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Crop
if len(borders) != 4:
log.error(
'ONNX Crop layer {} should take exactly 4 borders instead of {}'
.format(node.name, len(borders)))
return False
attrs = {'axis': int64_array([2, 3])}
if scale is not None:
attrs.update({
'dim': scale,
'offset': int64_array([borders[1], borders[0]])
})
else:
attrs.update({
'crop_begin': int64_array([borders[1], borders[0]]),
'crop_end': int64_array([borders[3], borders[2]])
})
Crop.update_node_stat(node, attrs)
return CropFrontExtractor.enabled
def test_crop_type1_infer_neg1(self):
graph = self._create_graph_type1()
crop_node = Node(graph, 'crop_node')
crop_node['axis'] = None
Crop.infer(crop_node)
self.assertIsNone(crop_node.out_node().shape)
def test_crop_type1_infer_neg2(self):
graph = self._create_graph_type1()
crop_node = Node(graph, 'crop_node')
crop_node['crop_begin'] = int64_array([1, 2, 3])
with self.assertRaisesRegex(Error, "number of crop_begin.*"):
Crop.infer(crop_node)
def test_crop_type3_infer_neg3(self):
graph = self._create_graph_type3()
crop_node = Node(graph, 'crop_node')
crop_node['offset'] = None
with self.assertRaisesRegex(Error, "offset attribute is missing.*"):
Crop.infer(crop_node)
def test_crop_type3_infer_neg3(self):
graph = self._create_graph_type3()
crop_node = Node(graph, 'crop_node')
crop_node['offset'] = None
Crop.infer(crop_node)
self.assertIsNone(crop_node.out_node().shape)
def test_crop_type1_infer_neg1(self):
graph = self._create_graph_type1()
crop_node = Node(graph, 'crop_node')
crop_node['axis'] = None
with self.assertRaisesRegex(Error, "axis attribute is missing .*"):
Crop.infer(crop_node)
def test_crop_type2_infer_neg1(self):
graph = self._create_graph_type2()
crop_node = Node(graph, 'crop_node')
crop_node['dim'] = int64_array([1, 2, 3])
with self.assertRaisesRegex(Error, "Number of axis.*"):
Crop.infer(crop_node)
def test_crop_type1_infer_neg2(self):
graph = self._create_graph_type1()
crop_node = Node(graph, 'crop_node')
crop_node['crop_begin'] = int64_array([1, 2, 3])
Crop.infer(crop_node)
self.assertIsNone(crop_node.out_node().shape)
def test_crop_type3_infer_neg4(self):
graph = self._create_graph_type3()
crop_node = Node(graph, 'crop_node')
crop_input2 = Node(graph, 'crop_input2')
crop_input2.shape = int64_array([1, 4, 423, 563])
Crop.infer(crop_node)
self.assertIsNone(crop_node.out_node().shape)
def test_crop_type3_infer_neg1(self):
graph = self._create_graph_type3()
crop_node = Node(graph, 'crop_node')
crop_input2 = Node(graph, 'crop_input2')
crop_input2.shape = None
Crop.infer(crop_node)
self.assertIsNone(crop_node.out_node().shape)
def test_crop_type2_infer_neg2(self):
graph = self._create_graph_type2()
crop_node = Node(graph, 'crop_node')
crop_node['dim'] = None
crop_node['crop_begin'] = None
Crop.infer(crop_node)
self.assertIsNone(crop_node.out_node().shape)
def test_crop_type3_infer_neg1(self):
graph = self._create_graph_type3()
crop_node = Node(graph, 'crop_node')
crop_input2 = Node(graph, 'crop_input2')
crop_input2.shape = None
with self.assertRaisesRegex(Error, "Not all input shapes were defined.*"):
Crop.infer(crop_node)
def extract(node):
attrs = get_mxnet_layer_attrs(node.symbol_dict)
axes = attrs.tuple("axes", int, [])
offset = [0 for i in range(0, axes[-1])]
node_attrs = {'axis': 1, 'offset': offset, 'dim': offset, 'axes': axes}
# update the attributes of the node
Crop.update_node_stat(node, node_attrs)
return __class__.enabled
def test_crop_type3_infer_neg4(self):
graph = self._create_graph_type3()
crop_node = Node(graph, 'crop_node')
crop_input2 = Node(graph, 'crop_input2')
crop_input2.shape = int64_array([1, 4, 423, 563])
with self.assertRaisesRegex(Error, "The crop for dimension is out of bounds.*"):
Crop.infer(crop_node)
def test_crop_type2_infer_neg2(self):
graph = self._create_graph_type2()
crop_node = Node(graph, 'crop_node')
crop_node['dim'] = None
crop_node['crop_begin'] = None
with self.assertRaisesRegex(Error, "Crop node crop_node should have either.*"):
Crop.infer(crop_node)
def extract(cls, node):
attrs = get_mxnet_layer_attrs(node.symbol_dict)
offset = attrs.tuple("offset", int, ())
axis = attrs.int("num_args", 0)
node_attrs = {
'axis': axis,
'offset': list(offset),
'dim': None,
}
Crop.update_node_stat(node, node_attrs)
return cls.enabled
def test_crop_type2_infer(self):
graph = self._create_graph_type2()
crop_node = Node(graph, 'crop_node')
Crop.infer(crop_node)
exp_shape = int64_array([1, 3, 100, 150])
res_shape = graph.node['crop_output']['shape']
self.assertTrue(np.array_equal(exp_shape, res_shape),
'shapes do not match expected: {} and given: {}'.format(exp_shape, res_shape))
def extract(cls, node):
pb = node.parameters
mapping_rule = {
'dim': pb['dim'],
'offset': pb['offset'],
'axis': pb['axis'],
'layout': 'NCHW'
}
Crop.update_node_stat(node, attrs=mapping_rule)
return cls.enabled
def extract(cls, node):
proto_layer = node.pb
param = proto_layer.crop_param
mapping_rule = {
'axis': param.axis,
'offset': param.offset,
'dim': None, # set in infer
'infer': crop_infer
}
# update the attributes of the node
Crop.update_node_stat(node, mapping_rule)
return cls.enabled
def replace_pattern(graph: Graph, match: dict):
node = match['op']
pair_node = Node(graph, node.pair_name)
if node.t >= 0:
raise Error('Does not support IfDefined with t > 0')
if node.in_port(0).get_source() is not None:
input_port = node.in_port(0).get_source()
op_output_id = node.out_port(0).get_destination().node.id
out_port = pair_node.out_port(0)
node_name = node.name
pair_name = pair_node.name
else:
input_port = pair_node.in_port(0).get_source()
op_output_id = pair_node.out_port(0).get_destination().node.id
out_port = node.out_port(0)
node_name = pair_node.name
pair_name = node.name
in_shape = input_port.data.get_shape()
node_t = abs(node.t)
init_value_memory_out = create_zero_value_with_batch_from_input(input_port, in_shape[1]*node_t)
memory_out = ReadValue(graph, {'name': pair_name, 'variable_id': node_name+pair_name}).create_node()
init_value_memory_out.out_port(0).connect(memory_out.in_port(0))
if node_t > 1:
crop_concat = Crop(graph, {'name': 'Memory_crop', 'dim': np.array([in_shape[1]*(node_t-1)]),
'offset': np.array([in_shape[1]]), 'axis': np.array([1])}).create_node()
memory_out.out_port(0).connect(crop_concat.in_port(0))
concat = Concat(graph, {'name': 'Memory_concat'}).create_node()
concat.add_sequence_of_ports('in', range(2))
crop_concat.out_port(0).connect(concat.in_port(0))
concat.in_port(1).connect(input_port)
memory_in = Assign(graph, {'name': node_name, 'variable_id': node_name + pair_name}).create_node()
concat.out_port(0).connect(memory_in.in_port(0))
out = Result(graph, {'name': 'Memory_output'}).create_node()
memory_in.out_port(0).connect(out.in_port(0))
crop_out = Crop(graph, {'name': 'Memory_crop_out', 'dim': np.array([in_shape[1]]),
'offset': np.array([0]), 'axis': np.array([1])}).create_node()
memory_out.out_port(0).connect(crop_out.in_port(0))
out_port.get_connection().set_source(crop_out.out_port(0))
else:
memory_in = Assign(graph, {'name': node_name, 'variable_id': node_name + pair_name}).create_node()
memory_in.in_port(0).connect(input_port)
out = Result(graph, {'name': 'Memory_output'}).create_node()
memory_in.out_port(0).connect(out.in_port(0))
out_port.get_connection().set_source(memory_out.out_port(0))
graph.remove_node(op_output_id)
graph.remove_node(node.id)
graph.remove_node(pair_node.id)
def replace_pattern(self, graph: nx.MultiDiGraph, match: dict):
node = match['slice']
# Caffe case
if not node.has_valid('start') or not node.has_valid('end'):
return
begin = node.start
end = node.end
input = node.in_node(0)
output_data = node.out_node()
# Check whether operation use only one axis or not
dims = 0
axes = np.zeros(begin.size)
for i in range(begin.size):
if begin[i] != 0 or end[i] != input.shape[i]:
dims += 1
axes[i] = 1
axes = np.array(axes, dtype=bool)
if dims == 0:
return
elif dims == 1:
# If Slice use only one axis, than
# convert Slice to StridedSlice
node['op'] = 'StridedSlice'
node['type'] = 'StridedSlice'
node['new_axis_mask'] = np.zeros(len(output_data.shape),
dtype=np.bool)
node['shrink_axis_mask'] = np.zeros(len(output_data.shape),
dtype=np.bool)
convert_negative_indices(begin, input.shape)
convert_negative_indices(end, input.shape)
else:
# If Slice use more than one axis use Crop layer
crop = Crop(
graph,
dict(axis=np.arange(begin.size)[axes], offset=begin[axes]))
# creating node with data
crop.create_node_with_data(inputs=[input],
data_nodes=[output_data])
# Remove unnecessary edges from and to to Slice vertex
graph.remove_edge(input.id, node.id)
graph.remove_edge(node.id, output_data.id)
def replace_op(self, graph: Graph, node: Node):
input_node = node.in_nodes()[0]
memory_pair_id = unique_id('id')
# Memory(in)
input_memory = Memory(
graph, {
'name':
'prev_splice_memory',
'id':
memory_pair_id,
'index':
1,
'size':
2,
'shape':
np.array(([input_node.shape[1] * len(node.context)]),
dtype=np.int64)
}).create_node()
# Memory(in) \
# Crop
# Input(temp) /
crop = Crop(
graph, {
'name':
'Splice_Crop',
'axis':
np.array([1], dtype=np.int64),
'offset':
np.array([input_node.shape[1]], dtype=np.int64),
'dim':
np.array([input_node.shape[1] * (len(node.context) - 1)],
dtype=np.int64)
}).create_node([input_memory])
# Crop \
# Concat
# Input /
concat_node = Concat(graph, {
'name': 'Splice_Concat',
'in_ports_count': 2,
'axis': 1
}).create_node([crop, input_node])
# Concat -> Memory(out)
Memory(
graph, {
'name':
'out_splice_memory',
'id':
memory_pair_id,
'index':
0,
'size':
2,
'shape':
np.array([input_node.shape[1] * len(node.context)],
dtype=np.int64)
}).create_node([concat_node])
return [concat_node.id]
def add_fake_background_loc(graph: Graph, input_node: Node):
"""
DetectionOutput layer expects that box coordinates contains coordinates of boxes for the "background" class also,
but in the TensorFlow\* Object Detection API the tensor contains information about real object classes only.
The function copies a slice of the output data of the node 'input_node' and then concats it to the beginning of the
data. The data in this slice is not used by the Detection Output layer so the actual values are not important. This
approach allows the model to be reshape-able and does not introduce many layers.
"background" class box coordinates.
:param graph: graph to operate on.
:param input_node: node producing the boxes coordinates.
:return convolution node that adds slice of data for the "background" class.
"""
crop_op = Crop(graph, dict(axis=np.array([1]), offset=np.array([0]), dim=np.array([1]), nchw_layout=True))
crop_node = crop_op.create_node([input_node], dict(name='crop_locs'))
concat_op = Concat(graph, dict(axis=1, in_ports_count=2, nchw_layout=True))
return concat_op.create_node([crop_node, input_node], dict(name=input_node.id + '/locs_with_fake_background'))
def replace_pattern(graph: Graph, match: dict):
node = match['op']
node_id = node['variable_id']
out_node_port = node.out_port(0).get_destination()
in_node_port = node.in_port(0).get_source()
node.in_port(0).disconnect()
node.out_port(0).disconnect()
crop = Crop(
graph, {
'name': 'Result_for_' + node_id,
'dim': np.array([1]),
'offset': np.array([0]),
'axis': np.array([0])
}).create_node()
in_node_port.connect(crop.in_port(0))
crop.out_port(0).connect(out_node_port)
def find_and_replace_pattern(self, graph: Graph):
for nms in graph.get_op_nodes(op='NonMaxSuppression'):
# prepare inputs to the NonMaximumSuppression Node
unsqueeze_boxes = create_op_node_with_second_input(
graph, Unsqueeze, int64_array([0]),
{'name': nms.soft_get('name') + '/Unsqueeze_0'})
nms.in_port(0).get_connection().insert_node(unsqueeze_boxes)
unsqueeze_box_scores = create_op_node_with_second_input(
graph, Reshape, int64_array([1, 1, -1]),
{'name': nms.soft_get('name') + '/Unsqueeze_1'})
nms.in_port(1).get_connection().insert_node(unsqueeze_box_scores)
nms_name = nms.soft_get('name', nms.id)
# prepare output #0
crop_box_indices_name = nms_name + '/Crop_boxes_'
crop_box_indices = Crop(
graph, {
'name': crop_box_indices_name,
'axis': int64_array([1]),
'offset': int64_array([2]),
'dim': int64_array([1])
}).create_node()
nms.out_port(0).get_connection().insert_node(crop_box_indices)
squeeze_output_boxes = create_op_node_with_second_input(
graph, Squeeze, int64_array([1]),
{'name': crop_box_indices_name + '/Squeeze'})
crop_box_indices.out_port(0).get_connection().insert_node(
squeeze_output_boxes)
num_of_outputs = len([
port for port in nms.out_ports().values()
if not port.disconnected()
])
if num_of_outputs == 1:
continue
# prepare output #1
crop_score_indices_name = nms_name + '/Crop_scores_'
crop_score_indices = Crop(
graph, {
'name': crop_score_indices_name,
'axis': int64_array([1]),
'offset': int64_array([2]),
'dim': int64_array([1])
}).create_node()
nms.out_port(1).get_connection().insert_node(crop_score_indices)
squeeze_output_scores = create_op_node_with_second_input(
graph, Squeeze, int64_array([1]),
{'name': crop_score_indices_name + '/Squeeze'})
crop_score_indices.out_port(0).get_connection().insert_node(
squeeze_output_scores)
def replace_pattern(graph: Graph, match: dict):
node = match['op']
node_id = node['id']
if node.in_port(0).disconnected():
i = 0
for dest in node.out_port(0).get_destinations():
new_in = Parameter(graph, {'name': "Parameter_"+str(i)+"_for_"+node_id,
'shape': dest.data.get_shape()}).create_node()
i += 1
dest.disconnect()
new_in.out_port(0).connect(dest)
log.error("Add input/output mapped {} -> {} ".format(new_in.name, "Result_for_"+node_id),
extra={'is_warning': True})
else:
out_node_port = node.out_port(0).get_destination()
in_node_port = node.in_port(0).get_source()
node.in_port(0).disconnect()
node.out_port(0).disconnect()
crop = Crop(graph, {'name': 'Result_for_'+node_id, 'dim': np.array([1]), 'offset': np.array([0]), 'axis': np.array([0])}).create_node()
in_node_port.connect(crop.in_port(0))
crop.out_port(0).connect(out_node_port)
def replace_pattern(graph: Graph, match: dict):
mem = match['op']
mem_shape = mem.in_port(0).data.get_shape()
mem_parent = mem.in_port(0).get_source()
context = mem['context']
for child_port in mem_parent.get_destinations():
child = child_port.node
# check if we find Splice containing context 'context'
if child['op'] == 'Splice' and child.id != mem.id and set(
child['context']).issubset(set(context)):
left_cont_out = child['context'][0]
left_cont = context[0]
for child_of_child in child.out_port(0).get_destinations():
out_transfer = child_of_child.node
out_transfer_port = child_of_child
if out_transfer['op'] == 'Crop':
# modify existing Crop to get right data from larger Splice
out_transfer['offset'] = out_transfer['offset'] + (
left_cont_out - left_cont) * mem_shape[-1]
else:
# insert Crop if we have not one
child_of_child.disconnect()
crop_node = Crop(
graph, {
'name':
graph.unique_id(prefix='Splice_crop_'),
'offset':
(left_cont_out - left_cont) * mem_shape[-1],
'dim':
np.array(
[len(child['context']) * mem_shape[-1]]),
'axis':
np.array([-1])
}).create_node()
child.out_port(0).connect(crop_node.in_port(0))
crop_node.out_port(0).connect(child_of_child)
crop_node.out_port(0).data.set_shape(
child.out_port(0).data.get_shape())
out_transfer_port = crop_node.in_port(0)
# move edge to child from old Splice to larger
out_transfer_port.disconnect()
mem.out_port(0).connect(out_transfer_port)
graph.remove_node(child.id)
def replace_sub_graph(graph: Graph, match: dict, **kwargs):
nms = match['nms']
# prepare inputs to the NonMaximumSuppression Node
unsqueeze_boxes = create_op_node_with_second_input(
graph, Unsqueeze, int64_array([0]),
{'name': nms.soft_get('name') + '/Unsqueeze_0'})
nms.in_port(0).get_connection().insert_node(unsqueeze_boxes)
unsqueeze_box_scores = create_op_node_with_second_input(
graph, Reshape, int64_array([1, 1, -1]),
{'name': nms.soft_get('name') + '/Unsqueeze_1'})
nms.in_port(1).get_connection().insert_node(unsqueeze_box_scores)
# prepare output
crop_box_indices = Crop(
graph, {
'name': nms.soft_get('name') + '/Crop',
'axis': int64_array([1]),
'offset': int64_array([2]),
'dim': int64_array([1])
}).create_node()
nms.out_port(0).get_connection().insert_node(crop_box_indices)
squeeze_output_boxes = create_op_node_with_second_input(
graph, Squeeze, int64_array([1]),
{'name': crop_box_indices.soft_get('name') + '/Squeeze'})
crop_box_indices.out_port(0).get_connection().insert_node(
squeeze_output_boxes)
if 5 in nms.in_ports() and not nms.in_port(5).disconnected():
soft_nms_sigma = nms.in_port(5).get_source().data.get_value()
if soft_nms_sigma is not None and soft_nms_sigma != 0.0:
log.error(
'The input to layer "{}" with value for the soft_nms_sigma is equal to "{}" but only value 0'
'is supported. The inference results will be incorrect.'.
format(nms.soft_get('name'), soft_nms_sigma))
def create_zero_value_with_batch_from_input(input_out_port: Port,
second_dim,
precision=np.float):
# create init_graph connected to ReadValue
graph = input_out_port.node.graph
input_name = input_out_port.node.name
shape_of_input = Shape(graph, {
'name': 'shape/' + input_name
}).create_node()
shape_of_input.in_port(0).connect(input_out_port)
dim_for_get_batch = Const(
graph, {
'name': 'dim/crop_batch/' + shape_of_input.name,
'value': int64_array([1]),
'shape': int64_array([1])
}).create_node()
get_batch = Crop(
graph, {
'name': 'crop_batch/' + shape_of_input.name,
'axis': int64_array([0]),
'offset': int64_array([0])
}).create_node()
get_batch.in_port(0).connect(shape_of_input.out_port(0))
get_batch.in_port(1).connect(dim_for_get_batch.out_port(0))
mem_shape_2nd_dim = Const(
graph, {
'name': 'gifo_r_weights_shape/' + input_name,
'value': int64_array([second_dim]),
'shape': int64_array([1])
}).create_node()
mem_shape = Concat(
graph, {
'name': 'gather_memory_shape/' + input_name,
'axis': 0,
'in_ports_count': 2
}).create_node()
mem_shape.in_port(0).connect(get_batch.out_port(0))
mem_shape.in_port(1).connect(mem_shape_2nd_dim.out_port(0))
fill_value = Const(
graph, {
'name': 'fill_value/' + input_name,
'value': np.array([0.0], precision),
'shape': int64_array([1])
}).create_node()
init_value_prev_lstm_output = Broadcast(graph, {
'name': 'init_value/' + input_name,
}).create_node()
init_value_prev_lstm_output.in_port(0).connect(fill_value.out_port(0))
init_value_prev_lstm_output.in_port(1).connect(mem_shape.out_port(0))
return init_value_prev_lstm_output
def replace_sub_graph(self, graph: Graph, match: dict):
slice_like = match['slice_like']
const = slice_like.in_nodes()[0]
crop_shape = slice_like.in_nodes()[1]
variants_dict = {
'mul_scalar1x': 0.1,
'mul_scalar2x': 0.2,
'mul_scalar1y': 0.1,
'mul_scalar2y': 0.2
}
for matches in find_pattern_matches(graph,
self.variants_pattern['nodes'],
self.variants_pattern['edges'],
None, None):
for k, v in matches.items():
if v in variants_dict.keys():
variants_dict[v] = Node(graph, k).in_nodes()[1].value[0]
variants = np.array([
variants_dict['mul_scalar1x'], variants_dict['mul_scalar1y'],
variants_dict['mul_scalar2x'], variants_dict['mul_scalar2y']
] * int(const.value.size / 4)).reshape(const.value.shape)
priorbox_variants = Const(
graph,
dict(value=variants,
symbol_dict={'name':
const.id + '/priorbox_variants'})).create_node()
variants_slice_like = Crop(graph, dict(axis=slice_like.axis, offset=slice_like.offset, dim=slice_like.dim, axes=slice_like.axes,
symbol_dict={'name': slice_like.id + '/variants_slice_like'})) \
.create_node()
variants_slice_like.in_port(0).connect(priorbox_variants.out_port(0))
variants_slice_like.in_port(1).connect(crop_shape.out_port(0))
concat = match['reshape3'].out_port(0).get_destination().node
assert concat.op == 'Concat'
concat_nodes_count = len(concat.in_nodes())
concat.add_input_port(concat_nodes_count)
concat.in_port(concat_nodes_count).get_connection().set_source(
variants_slice_like.out_port(0))
