def extract(cls, node):
proto_layer = node.pb
param = proto_layer.resample_param
types = [
"",
'nearest',
'linear',
'cubic',
'area',
]
resample_type = types[param.type]
update_attrs = {
'antialias': int(param.antialias),
'height': param.height,
'width': param.width,
'type': resample_type,
'factor': param.factor,
'fw': 'caffe',
}
mapping_rule = merge_attrs(param, update_attrs)
mapping_rule['mode'] = mapping_rule['type']
mapping_rule['axes'] = int64_array([2, 3])
mapping_rule.pop('type')
Interpolate.update_node_stat(node, mapping_rule)
return cls.enabled
def __call__(self, first: Node, second: Node) -> bool:
"""
This function checks whether Interpolate nodes 'first' and 'second' can be fused.
:param first: the first of fused nodes
:param second: the second of fused nodes
:return: True, if nodes can be fused, and False otherwise
"""
if not (is_next(first, second)
and self._compare_attributes(first, second)):
self.accumulated_axes = set()
return False
fst_axes = set([a for a in Interpolate.get_axes(first)])
snd_axes = set([a for a in Interpolate.get_axes(second)])
self.accumulated_axes = self.accumulated_axes | fst_axes
# If the set of accumulated axes and the set of axes of 'second' do not intersect then nodes can be fused,
# because interpolations with respect to various axes do not affect each other.
if not (self.accumulated_axes & snd_axes):
return True
# Otherwise, nodes cannot be fused.
self.accumulated_axes = set()
return False
def test_interpolate4_using_sizes(self, pads_begin, pads_end, input_shape, output_shape, sizes, scales, axes):
graph = build_graph(nodes_attrs=graph_nodes_attrs,
edges=graph_edges,
update_attributes={
'input_data': {'shape': input_shape},
'sizes': {'shape': int64_array(sizes).shape, 'value': int64_array(sizes)},
'sizes_data': {'shape': int64_array(sizes).shape, 'value': int64_array(sizes)},
'scales': {'shape': np.array(scales).shape, 'value': np.array(scales)},
'scales_data': {'shape': np.array(scales).shape, 'value': np.array(scales)},
'axes': {'shape': int64_array(axes).shape, 'value': int64_array(axes)},
'axes_data': {'shape': int64_array(axes).shape, 'value': int64_array(axes)},
'interpolate': {'pads_begin': int64_array(pads_begin),
'pads_end': int64_array(pads_end)}
})
node = Node(graph, 'interpolate')
tested_class = Interpolate(graph=graph, attrs=node.attrs())
tested_class.infer(node)
msg = "Interpolate-4 infer failed for case: sizes={}, scales={}, pads_begin={}, pads_end={}, axes={}," \
" expected_shape={}, actual_shape={}"
self.assertTrue(np.array_equal(graph.node['interpolate_data']['shape'], int64_array(output_shape)),
msg.format(sizes, scales, pads_begin, pads_end, axes, output_shape,
graph.node['interpolate_data']['shape']))
def replace_op(self, graph: Graph, node: Node):
mode = node.module.mode
if mode.endswith('linear'): # like bilinear or trilinear
mode = 'linear'
align_corners = node.module.align_corners
if mode == 'linear':
height = node.module.size[0] if node.module.size is not None else -1
width = node.module.size[1] if node.module.size is not None else -1
dims = node.module.dims
axes = np.arange(2, dims)
pads = np.zeros(dims, dtype=np.int32)
scales = np.repeat(node.module.scale_factor, dims - 2).astype(np.float32)
attrs = {
'name': node.name,
'version': 'opset4',
'height': height,
'width': width,
'mode': mode,
'axes': axes,
'pads_begin': pads,
'pads_end': pads,
'coordinate_transformation_mode': 'align_corners' if align_corners else 'half_pixel',
'shape_calculation_mode': 'sizes' if node.module.size is not None else 'scales',
}
sizes = Const(graph, {'value': np.array([height, width])}).create_node()
axes = Const(graph, {'value': axes}).create_node()
scales = Const(graph, {'value': scales}).create_node()
interp = Interpolate(graph, attrs).create_node([node.in_node(0), sizes, scales, axes])
else:
if node.module.size:
attrs = {
'name': node.name,
'version': 'opset1',
'height': node.module.size[0],
'width': node.module.size[1],
'mode': mode,
'axes': [2, 3],
'align_corners': node.module.align_corners,
}
interp = Interpolate(graph, attrs).create_node([node.in_node(0)])
else:
if not node.module.scale_factor:
raise Error('No scale_factor found')
attrs = {
'name': node.name,
'height_scale': np.float(node.module.scale_factor),
'width_scale': np.float(node.module.scale_factor),
'mode': mode,
'align_corners': node.module.align_corners,
}
interp = UpsampleOp(graph, attrs).create_node([node.in_node(0)])
return [interp.id]
def extract(cls, node):
attrs = get_mxnet_layer_attrs(node.symbol_dict)
scale = attrs.int("scale", 1)
num_filter = attrs.int("num_filter", 0)
mode = attrs.str("sample_type", None)
if mode == 'nearest':
node_attrs = {
'factor': attrs.int("scale", 1),
'mode': mode,
'antialias': 0,
'axes': int64_array([2, 3]),
}
Interpolate.update_node_stat(node, node_attrs)
elif mode == 'bilinear':
"""
Bilinear UpSampling uses deconvolution algorithm under the hood.
For MXNet Bilinear UpSampling op just wrapper over Deconvolution op.
Inputs data:
input1 - input data
input2 - deconvolution weight
"""
kernel = 2 * scale - scale % 2
stride = scale
pad = math.ceil((scale - 1) / 2)
num_group = num_filter
node_attrs = {
'op': __class__.op,
'type': 'Deconvolution',
'bias_addable': True,
'bias_term': False,
'pad': int64_array([[0, 0], [0, 0], [pad, pad], [pad, pad]]),
'pad_spatial_shape': int64_array([[pad, pad], [pad, pad]]),
'dilation': None,
'output_spatial_shape': None,
'output_shape': None,
'stride': int64_array([1, 1, stride, stride]),
'group': num_group,
'output': num_filter,
'kernel_spatial': int64_array([kernel, kernel]),
'input_feature_channel': 0,
'output_feature_channel': 1,
'kernel_spatial_idx': None,
'reshape_kernel': True,
'spatial_dims': None,
'channel_dims': int64_array([1]),
'batch_dims': int64_array([0]),
'layout': 'NCHW',
'get_pad': DeconvFrontExtractor.get_pad,
}
Convolution.update_node_stat(node, node_attrs)
return cls.enabled
def make_interpolate_reshape_able(self, interpolate: Node, concat: Node):
assert interpolate.soft_get('type') == 'Interpolate'
assert concat.soft_get('type') == 'Concat'
interp_axes = Interpolate.get_axes(interpolate)
concat_axis = self.get_concat_axis(concat)
if concat_axis is None or interp_axes is None \
or np.any(interp_axes < 0) or concat_axis < 0 \
or concat_axis in interp_axes:
# checks that interpolate axes and concat axis are valid and do not intersect
return
non_interp_concat_srcs = self.get_non_interpolate_concat_sources(
concat)
if not len(non_interp_concat_srcs):
# there is no Concat input to take input from
return
graph = interpolate.graph
src = non_interp_concat_srcs[0]
shape = Shape(graph, {
'name': src.node.soft_get('name', src.node.id) + '/Shape'
}).create_node()
shape.in_port(0).connect(src)
gather = create_op_with_const_inputs(
graph,
Gather, {
1: np.array(interp_axes, dtype=np.int32),
2: int64_array(0)
}, {'name': shape.name + '/Gathered'},
input_node=shape)
interpolate.in_port(1).get_connection().set_source(gather.out_port(0))
def make_interpolate_reshapeable(interpolate, concat):
assert interpolate.soft_get('type') == 'Interpolate'
assert concat.soft_get('type') == 'Concat'
output_shape = interpolate.out_port(0).data.get_shape()
interp_axes = [get_canonical_axis_index(output_shape, axis) for axis in Interpolate.get_axes(interpolate)]
concat_axis = get_canonical_axis_index(output_shape, concat.axis)
if concat_axis in interp_axes:
return
concat_srcs = [port.get_source() for port in concat.in_ports().values() if not port.disconnected()]
non_interp_concat_srcs = [src for src in concat_srcs if src.node.soft_get('type') != 'Interpolate']
if len(non_interp_concat_srcs) == 0:
return
graph = interpolate.graph
src = non_interp_concat_srcs[0]
shape = Shape(graph, {'name': src.node.soft_get('name', src.node.id) + '/Shape'}).create_node()
shape.in_port(0).connect(src)
gather = create_op_with_const_inputs(graph, Gather,
{1: np.array(interp_axes, dtype=np.int32), 2: int64_array(0)},
{'name': shape.name + '/Gathered'}, shape)
interpolate.in_port(1).get_connection().set_source(gather.out_port(0))
def extract(cls, node):
proto_layer = node.pb
param = proto_layer.interp_param
update_attrs = {
'height': param.height,
'width': param.width,
'zoom_factor': param.zoom_factor,
'shrink_factor': param.shrink_factor,
}
mapping_rule = merge_attrs(param, update_attrs)
mapping_rule.update({
'fw': 'caffe',
'mode': 'linear',
'axes': int64_array([2, 3]),
'pads_begin': param.pad_beg,
'pads_end': param.pad_end,
'align_corners': 1
})
Interpolate.update_node_stat(node, mapping_rule)
return cls.enabled
def replace_interpolate_pattern(graph: Graph, match: dict):
split = match['split']
scale = float32_array([get_split_scale(split)])
axis = int(split.in_port(1).get_connection().get_source().node.value)
split_node_name = split.name
axis_node = Const(graph, {'name': split_node_name + '/axis', 'value': int64_array([axis])}).create_node()
shape_node = Shape(graph, dict(name=split_node_name + '/Shape')).create_node()
scales_node = Const(graph, dict(name=split_node_name + '/scales', value=scale)).create_node()
mul_node = Mul(graph, dict(name=split_node_name + '/Mul')).create_node()
scales_node.out_port(0).connect(mul_node.in_port(1))
strided_slice_node = create_op_with_const_inputs(graph,
StridedSlice,
{1: int64_array([axis]), 2: int64_array([axis + 1])},
{
'name': split_node_name + '/StridedSlice',
'begin_mask': int64_array([1]),
'end_mask': int64_array([1]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])
})
shape_node.out_port(0).connect(strided_slice_node.in_port(0))
cast_shape_to_float = Cast(graph, {'dst_type': np.float32}).create_node()
strided_slice_node.out_port(0).connect(cast_shape_to_float.in_port(0))
cast_shape_to_float.out_port(0).connect(mul_node.in_port(0))
interp_node = Interpolate(graph,
dict(name=split_node_name + '/Interpolate',
mode='nearest',
antialias=0, pads_begin=int64_array([0]), pads_end=int64_array([0]),
coordinate_transformation_mode='half_pixel', nearest_mode='round_prefer_floor',
cube_coeff=-0.75, version='opset4', shape_calculation_mode='scales',
in_ports_count=4, maybe_part_of_sequence=True)).create_node()
floor_node = Floor(graph, {'name': split_node_name + '/Floor'}).create_node()
cast_mul_result_to_int = Cast(graph, {'dst_type': np.int64}).create_node()
mul_node.out_port(0).connect(floor_node.in_port(0))
floor_node.out_port(0).connect(cast_mul_result_to_int.in_port(0))
cast_mul_result_to_int.out_port(0).connect(interp_node.in_port(1))
scales_node.out_port(0).connect(interp_node.in_port(2))
axis_node.out_port(0).connect(interp_node.in_port(3))
match['concat'].out_port(0).get_connection().set_source(interp_node.out_port(0))
split_connection = split.in_port(0).get_connection()
split_connection.set_destination(interp_node.in_port(0))
split_connection.get_source().connect(shape_node.in_port(0))
def _compare_attributes_of_interpolate1(self, first: Node, second: Node) -> bool:
"""
This function checks whether attributes of Interpolate-1 nodes first and second are identical
(except attribute 'axes').
:param first: the first of compared nodes
:param second: the second of compared nodes
:return: True, if attributes of nodes are identical and False otherwise
"""
# If some of attributes 'mode', 'align_corners', 'antialias', 'pads_begin', 'pads_end' are different,
# then attributes of nodes are not identical.
op = Interpolate(graph=first.graph, attrs={})
for attr in ['mode', 'align_corners', 'antialias', 'pads_begin', 'pads_end']:
if first.soft_get(attr, default=op.attrs[attr]) != second.soft_get(attr, default=op.attrs[attr]):
return False
return True
def make_interpolate_reshapeable(interpolate):
assert interpolate.soft_get('type') == 'Interpolate'
axes = Interpolate.get_axes(interpolate)
input_shape = interpolate.in_port(0).data.get_shape()
output_shape = interpolate.out_port(0).data.get_shape()
if not np.all(np.remainder(output_shape, input_shape) == 0) and \
not np.all(np.remainder(input_shape, output_shape) == 0):
return
graph = interpolate.graph
name = interpolate.soft_get('name', interpolate.id)
shape = Shape(graph, {'name': name + '/ShapeOf'}).create_node()
shape.in_port(0).connect(interpolate.in_port(0).get_source())
gather = create_op_with_const_inputs(graph, Gather, {1: np.array(axes, dtype=np.int32), 2: int64_array(0)},
{'name': shape.name + '/Gathered'}, shape)
multipliers = output_shape[axes] / input_shape[axes]
mul = create_op_node_with_second_input(graph, Mul, multipliers, {'name': gather.name + '/Multiplied'}, gather)
interpolate.in_port(1).get_connection().set_source(mul.out_port(0))
def replace_sub_graph(self, graph: Graph, match: dict):
interpolate = match['interpolate']
transpose_1 = match['transpose_1']
transpose_2 = match['transpose_2']
axes = Interpolate.get_axes(interpolate)
if axes is None or not np.array_equal(axes, int64_array([1, 2])):
return
# because we remove Transpose layers the ResizeNearestNeighbor should be updated for NCHW layout
opset = interpolate.get_opset()
assert opset in ['opset1', 'opset4'], \
'Interpolate node with name {} has unsupported opset'.format(interpolate.soft_get('name', interpolate.id))
if opset == 'opset1':
interpolate.axes = int64_array([2, 3])
else:
interpolate.in_port(3).data.set_value(int64_array([2, 3]))
transpose_1.in_port(0).get_connection().set_destination(
interpolate.in_port(0))
transpose_2.out_port(0).get_connection().set_source(
interpolate.out_port(0))
graph.remove_nodes_from([transpose_1.id, transpose_2.id])
def replace_sequence(seq: List[Node], graph: Graph):
"""
This function replaces a sequence of consecutive Interpolate layers with one Interpolate layer,
if modes of all nodes of a sequence are the same.
:param seq: sequence of Interpolate layers
:param graph: graph to which nodes of seq belong
:return: Nothing
"""
if not seq:
return
if len(seq) == 1:
return
modes = set([n.mode for n in seq])
if len(modes) != 1:
return
dims_and_scales_ = []
# Each element of the list dims_and_scales_ is a pair
# (axis, output size for this axis) (opset1)
# or
# (axis, output size for this axis, output scales for this axis) (opset4)
if seq[0].get_opset() == 'opset1':
for interp in seq:
dims_and_scales_.extend(
zip(
Interpolate.get_axes(interp),
interp.in_port(
1).get_connection().get_source().data.get_value()))
axis_to_size = sorted(list(dict(dims_and_scales_).items()),
key=lambda x: x[0])
axes_of_node = int64_array([z[0] for z in axis_to_size])
sizes = shape_array([z[1] for z in axis_to_size])
scales = np.ones(len(axis_to_size), dtype=np.float32)
else:
for interp in seq:
dims_and_scales_.extend(
zip(
Interpolate.get_axes(interp),
interp.in_port(
1).get_connection().get_source().data.get_value(),
interp.in_port(
2).get_connection().get_source().data.get_value()))
axis_to_size = sorted(dims_and_scales_, key=lambda x: x[0])
axes_of_node = int64_array([z[0] for z in axis_to_size])
sizes = shape_array([z[1] for z in axis_to_size])
scales = mo_array([z[2] for z in axis_to_size])
fst_interp_node = seq[0]
last_interp_node = seq[-1]
last_interp_node_name = last_interp_node.soft_get('name',
last_interp_node.id)
attributes = get_interpolate_attributes(fst_interp_node)
opset = fst_interp_node.get_opset()
if opset == 'opset1':
attributes['axes'] = axes_of_node
interp_node = create_op_with_const_inputs(graph, Interpolate,
{1: sizes}, attributes)
fst_interp_connection = fst_interp_node.in_port(0).get_connection()
fst_interp_connection.set_destination(interp_node.in_port(0))
last_interp_node.out_port(0).get_connection().set_source(
interp_node.out_port(0))
else:
attributes['in_ports_count'] = 4
interp_node = create_op_with_const_inputs(graph, Interpolate, {
1: sizes,
2: scales,
3: axes_of_node
}, attributes)
fst_interp_connection = fst_interp_node.in_port(0).get_connection()
fst_interp_connection.set_destination(interp_node.in_port(0))
last_interp_node.out_port(0).get_connection().set_source(
interp_node.out_port(0))
rename_nodes([(last_interp_node, last_interp_node_name + '/delete'),
(interp_node, last_interp_node_name)])
def replace_pattern(self, graph: Graph, match: Dict[str, Node]):
log.debug('UpsampleToResample is triggered')
upsample = match['upsample']
upsample_name = upsample.soft_get('name', upsample.id)
input_shape = upsample.in_port(0).data.get_shape()
input_shape_rank = len(input_shape)
if input_shape_rank not in [4, 5]:
log.warning('The input shape is not 4D or 5D for op {}'.format(
upsample.soft_get('name')))
return
depth_scale = None
layout = graph.graph['layout']
if len(upsample.in_nodes()) == 2:
if upsample.in_node(1).value is None:
return
scales = upsample.in_node(1).value
assert len(scales) in (
4, 5
), 'Supported scales rank is 4 or 5, but it is {} for node {}'.format(
len(scales), upsample_name)
if not (math.isclose(scales[0], 1, rel_tol=1e-5)
and math.isclose(scales[1], 1, rel_tol=1e-5)):
return
height_scale = scales[get_height_dim(layout, input_shape_rank)]
width_scale = scales[get_width_dim(layout, input_shape_rank)]
if len(scales) == 5:
depth_scale = scales[get_depth_dim(layout, input_shape_rank)]
else:
height_scale = upsample['height_scale']
width_scale = upsample['width_scale']
if 1 in upsample.in_ports() and not upsample.in_port(1).disconnected():
upsample.in_port(1).disconnect()
upsample_name = upsample.soft_get('name', upsample.id)
shape = Shape(graph, {'name': upsample_name + '/0_port'}).create_node()
layout = graph.graph['layout']
if input_shape_rank == 4:
begin_value = int64_array(
[get_height_dim(layout, input_shape_rank)])
factor_value = float32_array([height_scale, width_scale])
else:
begin_value = int64_array(
[get_depth_dim(layout, input_shape_rank)])
factor_value = float32_array(
[depth_scale, height_scale, width_scale])
ss = create_op_with_const_inputs(
graph, StridedSlice, {
1: begin_value,
2: int64_array([get_width_dim(layout, input_shape_rank) + 1]),
3: int64_array([1])
}, {
'name': upsample_name + '/ss_0_port',
'begin_mask': int64_array([1]),
'end_mask': int64_array([1]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])
})
mul = create_op_node_with_second_input(
graph, Mul, factor_value, {'name': upsample_name + '/factor_mul'})
source = upsample.in_port(0).get_connection().get_source()
source.connect(shape.in_port(0))
shape.out_port(0).connect(ss.in_port(0))
ss.out_port(0).connect(mul.in_port(0))
# Create Interpolate operation
if input_shape_rank == 4:
axes = int64_array([
get_height_dim(layout, input_shape_rank),
get_width_dim(layout, input_shape_rank)
])
else:
axes = int64_array([
get_depth_dim(layout, input_shape_rank),
get_height_dim(layout, input_shape_rank),
get_width_dim(layout, input_shape_rank)
])
axes_node = Const(graph, {
'name': upsample_name + '/axis',
'value': axes
}).create_node()
interpolate = Interpolate(
graph, {
'mode': upsample.attrs()['mode'],
'antialias': 0,
'pads_begin': int64_array([0]),
'pads_end': int64_array([0]),
'coordinate_transformation_mode': 'half_pixel',
'nearest_mode': 'round_prefer_floor',
'cube_coeff': -0.75,
'shape_calculation_mode': 'scales',
'version': 'opset4',
'in_ports_count': 4
}).create_node()
upsample.add_input_port(1, skip_if_exist=True)
assert upsample.in_port(1).disconnected()
mul.out_port(0).connect(interpolate.in_port(1))
axes_node.out_port(0).connect(interpolate.in_port(3))
scales_node = Const(graph, {
'name': upsample_name + '/scales',
'value': factor_value
}).create_node()
scales_node.out_port(0).connect(interpolate.in_port(2))
upsample.in_port(0).get_connection().set_destination(
interpolate.in_port(0))
upsample.out_port(0).get_connection().set_source(
interpolate.out_port(0))
rename_nodes([(upsample, upsample_name + '/delete'),
(interpolate, upsample_name)])
convert_to_float = Cast(graph, dict(dst_type=np.float32)).create_node()
convert_to_int = Cast(graph, dict(dst_type=np.int64)).create_node()
mul.in_port(0).get_connection().insert_node(convert_to_float)
mul.out_port(0).get_connection().insert_node(convert_to_int)
def replace_resize(graph: Graph, resize: Node):
log.debug("Converting of ONNX Resize-10 to Interpolate-4 "
"is triggered for node {}.".format(
resize.soft_get('name', resize.id)))
resize_name = resize.soft_get('name', resize.id)
rank_node = Rank(graph, {'name': resize_name + '/max_axes'}).create_node()
range_node = create_op_with_const_inputs(graph, Range, {
0: int64_array(2),
2: int64_array(1)
}, {'name': resize_name + '/axes'})
sizes_ss = create_op_with_const_inputs(graph, StridedSlice, {
1: int64_array([2]),
2: int64_array([0]),
3: int64_array([1])
}, {
'name': resize_name + '/sizes_ss',
'begin_mask': int64_array([1]),
'end_mask': int64_array([0]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])
})
scales_ss = create_op_with_const_inputs(
graph, StridedSlice, {
1: int64_array([2]),
2: int64_array([0]),
3: int64_array([1])
}, {
'name': resize_name + '/scales_ss',
'begin_mask': int64_array([1]),
'end_mask': int64_array([0]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])
})
rank_node.out_port(0).connect(range_node.in_port(1))
interpolate_node = Interpolate(
graph, {
'version': 'opset4',
'mode': 'linear_onnx' if resize.mode == 'linear' else 'nearest',
'coordinate_transformation_mode': 'asymmetric',
'cube_coeff': -0.75,
'nearest_mode': 'simple',
'pads_begin': int64_array([0]),
'pads_end': int64_array([0]),
'antialias': 0,
'shape_calculation_mode': 'scales',
'in_ports_count': 4
}).create_node()
range_node.out_port(0).connect(interpolate_node.in_port(3))
shape_of = Shape(graph, {'name': resize_name + '/ShapeOf'}).create_node()
# When we calculate 'sizes' input as floor(input_shape * scales), we can get incorrect 'sizes' if, e.g.,
# scales = [1.0, 1.0, 1.33333, 2.0], input_shape = [1, 3, 30, 200], because
# input_shape * scales = [1, 3, 39.9999, 400], and floor(input_shape * scales)[2] == 39, not 40.
# Maybe we need to calculate 'sizes' input as floor(input_shape * scales + eps), where eps is some small
# floating point number, e.g. 1.0e-5. But, in this case, if scales = [1.0, 1.0, 1.333333, 2.0],
# input_shape = [1, 3, 30, 200], floor(input_shape * scales + eps) = 39, not 40, because
# input_shape[2] * scales[2] + 1.0e-5 = 39.99991.
# Hence, we need to calculate 'sizes' as floor(input_shape * (scales + eps)).
add_node = create_op_with_const_inputs(graph, Add,
{1: float_array([1.0e-5])},
{'name': resize_name + '/Add'})
dst_dtype = np.float32 # even if data_type=FP16 use float32 for shape values
cast_shape_to_float = Cast(graph, {'dst_type': dst_dtype}).create_node()
shape_of.out_port(0).connect(cast_shape_to_float.in_port(0))
mul_node = Mul(graph, {
'name': resize_name + '/Mul'
}).create_node([cast_shape_to_float, add_node])
floor_node = Floor(graph, {
'name': resize_name + '/Floor'
}).create_node([mul_node])
cast_mul_result_to_int = Cast(graph, {
'dst_type': np.int64
}).create_node([floor_node])
cast_mul_result_to_int.out_port(0).connect(sizes_ss.in_port(0))
sizes_ss.out_port(0).connect(interpolate_node.in_port(1))
scales_ss.out_port(0).connect(interpolate_node.in_port(2))
connection_of_resize_input = resize.in_port(0).get_connection()
connection_of_resize_input.set_destination(interpolate_node.in_port(0))
connection_of_scales = resize.in_port(1).get_connection()
connection_of_scales.set_destination(scales_ss.in_port(0))
connection_of_resize_input.get_source().connect(shape_of.in_port(0))
connection_of_resize_input.get_source().connect(rank_node.in_port(0))
connection_of_scales.get_source().connect(add_node.in_port(0))
rename_nodes([(resize, resize_name + '/delete'),
(interpolate_node, resize_name)])
resize.out_port(0).get_connection().set_source(
interpolate_node.out_port(0))
def replace_sub_graph(self, graph: Graph, match: dict):
log.debug('Matched NearestNeighborUpsampling pattern: {}'.format(
[node.id for node in match.values()]))
try:
input_height = match['pack_1'].in_node(1).value.item()
input_width = match['pack_1'].in_node(3).value.item()
height_scale = match['mul_const'].shape[-4]
width_scale = match['mul_const'].shape[-2]
except Exception as ex:
log.warning(
'Failed to determine scaling parameters from the topology. Do not apply pattern.'
)
return
reshape2_name = match['reshape_2'].name
resample_op = Interpolate(
graph, {
'mode': 'nearest',
'antialias': 0,
'pads_begin': int64_array([0]),
'pads_end': int64_array([0]),
'coordinate_transformation_mode': 'half_pixel',
'nearest_mode': 'round_prefer_floor',
'cube_coeff': -0.75,
'version': 'opset4',
'name': reshape2_name + '/Resample',
'shape_calculation_mode': 'scales',
'in_ports_count': 4
})
resample_node = resample_op.create_node([match['op']])
axes_node = Const(
graph, {
'name':
resample_node.name + '/axes',
'value':
int64_array([2, 3])
if graph.graph['layout'] == 'NCHW' else int64_array([1, 2])
}).create_node()
sizes_node = Const(
graph, {
'value':
mo_array(
[input_height * height_scale, input_width * width_scale]),
'name':
resample_node.name + '/target_shape'
}).create_node()
scales_node = Const(
graph, {
'value': float32_array([height_scale, width_scale]),
'name': resample_node.name + '/scales'
}).create_node()
match['reshape_2'].replace_node(resample_node)
resample_node.add_input_port(1, skip_if_exist=True)
assert resample_node.in_port(1).disconnected()
sizes_node.out_port(0).connect(resample_node.in_port(1))
scales_node.out_port(0).connect(resample_node.in_port(2))
axes_node.out_port(0).connect(resample_node.in_port(3))
graph.remove_nodes_from(
[node.id for node in match.values() if node.id != match['op'].id])
def replace_resize(graph: Graph, resize: Node):
log.debug("Converting of ONNX Resize-11 to Interpolate-4 "
"is triggered for node {}.".format(
resize.soft_get('name', resize.id)))
input_shape = resize.in_port(0).data.get_shape()
input_rank = len(input_shape)
resize_name = resize.soft_get('name', resize.id)
if input_rank not in {4, 5}:
log.warning(
'The input shape is not 4D or 5D for op with name {}'.format(
resize_name))
return
assert (resize.is_in_port_connected(0) and (resize.is_in_port_connected(2) or resize.is_in_port_connected(3))), \
"Scales or sizes inputs must be connected to Node {} with op {}.".format(resize.soft_get("name", resize.id),
resize.op)
assert resize.soft_get('coordinate_transformation_mode') != 'tf_crop_and_resize', \
'Mode tf_crop_and_resize is not supported for op {} with name {}'.format(resize.op,
resize.soft_get("name", resize.id))
layout = graph.graph['layout']
if input_rank == 4:
begin_dim = get_height_dim(layout, input_rank)
end_dim = get_width_dim(layout, input_rank) + 1
else:
begin_dim = get_depth_dim(layout, input_rank)
end_dim = get_width_dim(layout, input_rank) + 1
sizes_ss = create_op_with_const_inputs(
graph, StridedSlice, {
1: int64_array([begin_dim]),
2: int64_array([end_dim]),
3: int64_array([1])
}, {
'name': resize_name + '/StridedSlice_sizes',
'begin_mask': int64_array([1]),
'end_mask': int64_array([1]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])
})
scales_ss = create_op_with_const_inputs(
graph, StridedSlice, {
1: int64_array([begin_dim]),
2: int64_array([end_dim]),
3: int64_array([1])
}, {
'name': resize_name + '/StridedSlice_scales',
'begin_mask': int64_array([1]),
'end_mask': int64_array([1]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0])
})
axes_node = Const(
graph, {
'name': resize_name + '/axis',
'value': int64_array(np.arange(begin_dim, end_dim))
}).create_node()
shape_calculation_mode = 'sizes' if resize.is_in_port_connected(
3) else 'scales'
interpolate_node = Interpolate(
graph, {
'version': 'opset4',
'mode': convert_mode(resize.mode),
'coordinate_transformation_mode':
resize.coordinate_transformation_mode,
'cube_coeff': resize.cube_coeff,
'nearest_mode': resize.nearest_mode,
'pads_begin': int64_array([0]),
'pads_end': int64_array([0]),
'antialias': 0,
'shape_calculation_mode': shape_calculation_mode,
'in_ports_count': 4
}).create_node()
axes_node.out_port(0).connect(interpolate_node.in_port(3))
shape_of = Shape(graph, {'name': resize_name + '/ShapeOf'}).create_node()
add_node = create_op_with_const_inputs(graph, Add,
{1: float_array([1.0e-5])},
{'name': resize_name + '/Add'})
dst_dtype = np.float32 # even if data_type=FP16 use float32 for shape values
if not resize.is_in_port_connected(3):
cast_shape_to_float = Cast(graph, {
'dst_type': dst_dtype
}).create_node()
mul_node = Mul(graph, {'name': resize_name + '/Mul'}).create_node()
shape_of.out_port(0).connect(cast_shape_to_float.in_port(0))
cast_shape_to_float.out_port(0).connect(mul_node.in_port(0))
cast_add_result_to_int = Cast(graph, {
'dst_type': np.int64
}).create_node()
floor_node = Floor(graph, {
'name': resize_name + '/Floor'
}).create_node()
mul_node.out_port(0).connect(add_node.in_port(0))
add_node.out_port(0).connect(floor_node.in_port(0))
floor_node.out_port(0).connect(cast_add_result_to_int.in_port(0))
cast_add_result_to_int.out_port(0).connect(sizes_ss.in_port(0))
sizes_ss.out_port(0).connect(interpolate_node.in_port(1))
scales_ss.out_port(0).connect(interpolate_node.in_port(2))
connection_of_resize_input = resize.in_port(0).get_connection()
connection_of_resize_input.set_destination(interpolate_node.in_port(0))
connection_of_scales = resize.in_port(2).get_connection()
connection_of_scales.set_destination(scales_ss.in_port(0))
connection_of_resize_input.get_source().connect(shape_of.in_port(0))
connection_of_scales.get_source().connect(mul_node.in_port(1))
else:
cast_shape_to_float = Cast(graph, {
'dst_type': dst_dtype
}).create_node()
cast_sizes_to_float = Cast(graph, {
'dst_type': dst_dtype
}).create_node()
div_node = Div(graph, {'name': resize_name + '/Div'}).create_node()
cast_sizes_to_float.out_port(0).connect(div_node.in_port(0))
cast_shape_to_float.out_port(0).connect(div_node.in_port(1))
shape_of.out_port(0).connect(cast_shape_to_float.in_port(0))
div_node.out_port(0).connect(add_node.in_port(0))
add_node.out_port(0).connect(scales_ss.in_port(0))
scales_ss.out_port(0).connect(interpolate_node.in_port(2))
sizes_ss.out_port(0).connect(interpolate_node.in_port(1))
connection_of_resize_input = resize.in_port(0).get_connection()
connection_of_resize_input.set_destination(interpolate_node.in_port(0))
connection_of_sizes = resize.in_port(3).get_connection()
connection_of_sizes.set_destination(sizes_ss.in_port(0))
connection_of_resize_input.get_source().connect(shape_of.in_port(0))
connection_of_sizes.get_source().connect(
cast_sizes_to_float.in_port(0))
rename_nodes([(resize, resize_name + '/delete'),
(interpolate_node, resize_name)])
resize.out_port(0).get_connection().set_source(
interpolate_node.out_port(0))
