def Mean(onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Calculate element-wise mean of the input tensors."""
initial_value_node = ng.constant(
0, get_dtype(ng_inputs[0].get_element_type()))
sum_node = reduce(ng.add, ng_inputs, initial_value_node)
count_array = np.full(sum_node.shape,
len(ng_inputs),
dtype=get_dtype(sum_node.get_element_type()))
return sum_node / ng.constant(count_array)
def make_convolution_op(onnx_node, ng_inputs):
# type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""
Create an ngraph convolution Op based on an ONNX node.
:param onnx_node: wrapped ONNX node for Conv of ConvTranspose op
:param ng_inputs: ngraph TensorOp input tensors
:return: ngraph Op for convolution or deconvolution
"""
if len(ng_inputs) == 3:
x, weights, bias = ng_inputs
elif len(ng_inputs) == 2:
x, weights = ng_inputs
bias = ng.constant(0, dtype=get_dtype(x.get_element_type()))
else:
raise ValueError(
'Conv node (%s): unexpected number of input values: %d.',
onnx_node.name, len(ng_inputs))
groups = onnx_node.get_attribute_value('group', 1)
if groups != 1:
log.warning(
'Conv node (%s): `group` attribute value %d is not supported.',
onnx_node.name, groups)
strides = get_strides(onnx_node)
dilation = get_dilations(onnx_node)
padding_below, padding_above = get_pads(onnx_node)
conv = ng.convolution(x, weights, strides, dilation, padding_below,
padding_above)
return conv + bias
def as_elementwise_compatible_nodes(*input_values): # type: (*NodeInput) -> List[Node]
"""Return all input values as ngraph Nodes with the same shape and element type.
Scalar values will be converted to ngraph Constant Nodes.
"""
input_nodes = [node for node in input_values
if issubclass(type(node), Node)] # type: List[Node]
if not input_nodes:
raise NotImplementedError('Operations on scalars only are not supported.')
shapes = {tuple(node.shape) for node in input_nodes}
if len(shapes) > 1:
log.warning('More than one different shape in input nodes %s.', input_nodes)
types = [node.get_element_type() for node in input_nodes]
unique_types = {repr(type) for type in types}
if len(unique_types) > 1:
log.warning('More than one different data type in input nodes %s.', input_nodes)
sorted_shapes = sorted(shapes, key=len)
broadcast_shape = sorted_shapes.pop()
broadcast_dtype = get_dtype(types.pop())
output_nodes = []
for input_value in input_values:
if issubclass(type(input_value), Node):
input_value = ng.broadcast(input_value, broadcast_shape)
output_nodes.append(input_value)
else:
input_value = make_constant_node(input_value, dtype=broadcast_dtype)
output_nodes.append(ng.broadcast(input_value, broadcast_shape))
return output_nodes
def BatchNormalization(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Carry out batch normalization."""
x, scale, bias, mean, var = ng_inputs
is_test = onnx_node.get_attribute_value('is_test', 1)
spatial = onnx_node.get_attribute_value('spatial', 1)
epsilon = onnx_node.get_attribute_value('epsilon', 1e-3)
# @TODO: Implement learning mode support
# momentum = onnx_node.get_attribute_value('momentum', 0.99)
if not is_test:
raise NotImplementedError('BatchNormalization node (%s): only `is_test` mode is currently '
'supported.', onnx_node.name)
if not spatial:
raise NotImplementedError('BatchNormalization node (%s): only `spatial` mode is currently '
'supported.', onnx_node.name)
mean = ng.broadcast(mean, x.shape, axis=1)
scale = ng.broadcast(scale, x.shape, axis=1)
var = ng.broadcast(var, x.shape, axis=1)
bias = ng.broadcast(bias, x.shape, axis=1)
epsilon = ng.broadcast(ng.constant(epsilon, dtype=get_dtype(x.get_element_type())),
x.shape, axis=1)
return (scale * ((x - mean) * (1 / (ng.sqrt(var + epsilon)))) + bias)
def Cast(onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Limit input tensor values within specified interval."""
data = ng_inputs[0]
cast_to_type = onnx_node.get_attribute_value('to')
if cast_to_type is None:
raise ValueError('Cast node (%s): \'to\' attribute is required.')
input_tensor_type = get_dtype(data.get_element_type())
new_type = onnx_tensor_type_to_numpy_type(cast_to_type)
unsupported_types = [
onnx_tensor_type_to_numpy_type('COMPLEX64'),
onnx_tensor_type_to_numpy_type('COMPLEX128'),
]
if input_tensor_type in unsupported_types:
raise ValueError(
'Cast node (%s): input tensor data type (%s) is not supported.',
onnx_node.name, str(input_tensor_type))
if new_type in unsupported_types:
raise ValueError(
'Cast node (%s): casting to type (%s) is not supported.',
onnx_node.name, str(new_type))
return ng.convert(data, new_type)
def as_elementwise_compatible_nodes(*input_values): # type: (*NodeInput) -> List[Node]
"""Return all input values as ngraph Nodes with the same shape and element type.
Scalar values will be converted to ngraph Constant Nodes.
"""
input_nodes = [node for node in input_values
if issubclass(type(node), Node)] # type: List[Node]
if not input_nodes:
raise NotImplementedError('Operations on scalars only are not supported.')
shapes = {tuple(node.shape) for node in input_nodes}
if len(shapes) > 1:
log.warning('More than one different shape in input nodes %s.', input_nodes)
types = [node.get_element_type() for node in input_nodes]
unique_types = {repr(type) for type in types}
if len(unique_types) > 1:
log.warning('More than one different data type in input nodes %s.', input_nodes)
sorted_shapes = sorted(shapes, key=len)
broadcast_shape = sorted_shapes.pop()
broadcast_dtype = get_dtype(types.pop())
output_nodes = []
for input_value in input_values:
if issubclass(type(input_value), Node):
input_value = ng.broadcast_to(input_value, broadcast_shape)
output_nodes.append(input_value)
else:
input_value = make_constant_node(input_value, dtype=broadcast_dtype)
output_nodes.append(ng.broadcast_to(input_value, broadcast_shape))
return output_nodes
def ReduceMean(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Compute the mean value of the input tensor's elements along the provided axes."""
input_shape = list(ng_inputs[0].shape)
sum_node = make_reduction_op(ng.sum, onnx_node, ng_inputs[0])
reduction_axes = get_reduction_axes(onnx_node, ng_inputs[0])
avg_elem_count = np.prod([input_shape[x] for x in reduction_axes])
const_node = ng.broadcast(ng.constant(avg_elem_count, get_dtype(sum_node.get_element_type())),
sum_node.shape)
return ng.divide(sum_node, const_node)
def ConvTranspose(
onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Calculate convolution transpose."""
if len(ng_inputs) == 3:
data, weights, bias = ng_inputs
elif len(ng_inputs) == 2:
data, weights = ng_inputs
bias = ng.constant(0, dtype=get_dtype(data.get_element_type()))
strides = get_strides(onnx_node)
dilation = get_dilations(onnx_node)
padding_below, padding_above = get_pads(onnx_node)
output_padding = onnx_node.get_attribute_value('output_padding')
if output_padding is None:
raise ValueError(
'ConvTranspose node (s%): output_padding attribute is required.',
onnx_node.name)
data_shape = list(data.shape)
weights_shape = list(weights.shape)
num_spatial_dims = len(data.shape) - 2
data_dilation_strides = [1, 1]
data_batch_shape = [1] * (num_spatial_dims + 2)
data_batch_shape[0] = data_shape[0]
data_batch_shape[1] = weights_shape[1]
for i in range(num_spatial_dims):
# Calculating spatial dims of data output shape for ngraph conv backprop op
# | pb + s(ds-1) + op - d(ws-1)+1 |
# | ----------------------------- | + 1
# |_ dds _|
#
# d - dilation
# ds - data shape
# dds - data dilation strides
# op - putput padding
# pb - padding below
# s - strides
# ws - weights shape
data_batch_shape[i + 2] = (
(padding_below[i] +
((data_shape[i + 2] - 1) * strides[i] + 1) + output_padding[i]) -
((weights_shape[i + 2] - 1) * dilation[i] + 1) +
1) // data_dilation_strides[i] + 1
transconv = ng.convolution_backprop_data(data_batch_shape, weights, data,
strides, dilation, padding_below,
padding_above,
data_dilation_strides)
if len(bias.shape) > 0:
return transconv + ng.broadcast_to(bias, transconv.shape, 1)
else:
return transconv
def __call__(self, *input_values: NumericData) -> List[NumericData]:
"""Run computation on input values and return result."""
input_values = [np.array(input_value) for input_value in input_values]
input_shapes = [get_shape(input_value) for input_value in input_values]
param_names = [param.friendly_name for param in self.parameters]
if self.network_cache.get(str(input_shapes)) is None:
capsule = Function.to_capsule(self.function)
cnn_network = IENetwork(capsule)
if self.function.is_dynamic():
cnn_network.reshape(dict(zip(param_names, input_shapes)))
# Convert unsupported inputs of the network
_convert_inputs(cnn_network)
self.network_cache[str(input_shapes)] = cnn_network
else:
cnn_network = self.network_cache[str(input_shapes)]
executable_network = self.runtime.backend.load_network(
cnn_network, self.runtime.backend_name)
# Input validation
if len(input_values) != len(self.parameters):
raise UserInputError("Expected %s parameters, received %s.",
len(self.parameters), len(input_values))
for parameter, input in zip(self.parameters, input_values):
parameter_shape = parameter.get_output_partial_shape(0)
input_shape = PartialShape(input.shape)
if len(input.shape) > 0 and not parameter_shape.compatible(
input_shape):
raise UserInputError(
"Provided tensor's shape: %s does not match the expected: %s.",
input_shape,
parameter_shape,
)
request = executable_network.requests[0]
request.infer(dict(zip(param_names, input_values)))
# Set order of output blobs compatible with nG Function
result_buffers = [
self.__get_ie_output_blob_buffer(request.output_blobs, result)
for result in self.results
]
# Since OV overwrite result data type we have to convert results to the original one.
original_dtypes = [
get_dtype(result.get_output_element_type(0))
for result in self.results
]
converted_buffers = [
buffer.astype(original_dtype)
for buffer, original_dtype in zip(result_buffers, original_dtypes)
]
return converted_buffers
def ThresholdedRelu(
onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Apply the Thresholded Relu function to the input tensor elementwise.
f(x) = 0 for x <= alpha, f(x) = x for x > alpha
"""
x = ng_inputs[0]
alpha = onnx_node.get_attribute_value('alpha', 1.0)
x_map = ng.convert(ng.greater(x, alpha), get_dtype(x.get_element_type()))
x = x * x_map
return x
def Clip(onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Limit input tensor values within specified interval."""
data = ng_inputs[0]
data_elem_dtype = get_dtype(data.get_element_type())
max_value = onnx_node.get_attribute_value('max',
np.finfo(data_elem_dtype).max)
min_value = onnx_node.get_attribute_value('min',
np.finfo(data_elem_dtype).min)
return ng.minimum(
ng.maximum(data, ng.constant(min_value, data_elem_dtype)),
ng.constant(max_value, data_elem_dtype))
def _write_ndarray_to_tensor_view(value, tensor_view):
# type: (np.ndarray, TensorViewType) -> None
tensor_view_dtype = get_dtype(tensor_view.element_type)
if value.dtype != tensor_view_dtype:
log.warning(
'Attempting to write a %s value to a %s tensor. Will attempt type conversion.',
value.dtype,
tensor_view.element_type)
value = value.astype(tensor_view_dtype)
buffer_size = Computation._get_buffer_size(
tensor_view.element_type, tensor_view.element_count)
tensor_view.write(util.numpy_to_c(np.ascontiguousarray(value)), 0, buffer_size)
def _write_ndarray_to_tensor_view(value, tensor_view):
# type: (np.ndarray, TensorViewType) -> None
tensor_view_dtype = get_dtype(tensor_view.element_type)
if value.dtype != tensor_view_dtype:
log.warning(
'Attempting to write a %s value to a %s tensor. Will attempt type conversion.',
value.dtype,
tensor_view.element_type)
value = value.astype(tensor_view_dtype)
buffer_size = Computation._get_buffer_size(
tensor_view.element_type, tensor_view.element_count)
tensor_view.write(util.numpy_to_c(np.ascontiguousarray(value)), 0, buffer_size)
def HardSigmoid(
onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Apply f(x) = max(0, min(1, alpha * x + beta)) function to tensor element-wise.
:param onnx_node: The ONNX node representing this operation.
:param ng_inputs: The input tensors.
:return: The tensor with applied HardSigmoid operation.
"""
data = ng_inputs[0]
data_type = get_dtype(data.get_element_type()).type
alpha = onnx_node.get_attribute_value('alpha', float(0.2))
beta = onnx_node.get_attribute_value('beta', float(0.5))
return ng.maximum(data_type(0),
ng.minimum(data_type(1), alpha * data + beta))
def __call__(self, *input_values): # type: (*NumericData) -> List[NumericData]
"""Run computation on input values and return result."""
for tensor_view, value in zip(self.tensor_views, input_values):
if not isinstance(value, np.ndarray):
value = np.array(value)
Computation._write_ndarray_to_tensor_view(value, tensor_view)
self.handle.call(self.result_views, self.tensor_views)
results = []
for result_view in self.result_views:
result = np.ndarray(result_view.shape, dtype=get_dtype(result_view.element_type))
Computation._read_tensor_view_to_ndarray(result_view, result)
results.append(result)
return results
def get_bool_nodes(
nodes): # type: (Tuple[NgraphNode, ...]) -> Tuple[NgraphNode, ...]
"""Convert each input node to bool data type if necessary.
:param nodes: Input nodes to be converted.
:return: Converted nodes.
"""
bool_nodes = []
for node in nodes:
if not node.get_element_type() == NgraphType.boolean:
bool_nodes.append(ng.convert(node, bool))
logger.warning('Converting node of type: <{}> to bool.'.format(
get_dtype(node.get_element_type())))
else:
bool_nodes.append(node)
return tuple(bool_nodes)
def _write_ndarray_to_tensor_view(value, tensor_view):
# type: (np.ndarray, TensorViewType) -> None
tensor_view_dtype = get_dtype(tensor_view.element_type)
if list(tensor_view.shape) != list(value.shape) and len(
value.shape) > 0:
raise UserInputError(
'Provided tensor\'s shape: %s does not match the expected: %s.',
list(value.shape), list(tensor_view.shape))
if value.dtype != tensor_view_dtype:
log.warning(
'Attempting to write a %s value to a %s tensor. Will attempt type conversion.',
value.dtype, tensor_view.element_type)
value = value.astype(tensor_view_dtype)
buffer_size = Computation._get_buffer_size(tensor_view.element_type,
tensor_view.element_count)
tensor_view.write(util.numpy_to_c(np.ascontiguousarray(value)), 0,
buffer_size)
def __call__(self, *input_values): # type: (*NumericData) -> NumericData
"""Run computation on input values and return result."""
for tensor_view, value in zip(self.tensor_views, input_values):
if not isinstance(value, np.ndarray):
value = np.array(value)
Computation._write_ndarray_to_tensor_view(value, tensor_view)
result_element_type = self.function.get_output_element_type(0)
result_shape = self.function.get_output_shape(0)
result_dtype = get_dtype(result_element_type)
result_view = self.runtime.backend.create_tensor(result_element_type, result_shape)
result_arr = np.empty(result_shape, dtype=result_dtype)
self.runtime.backend.call(self.function, [result_view], self.tensor_views)
Computation._read_tensor_view_to_ndarray(result_view, result_arr)
result_arr = result_arr.reshape(result_shape)
return result_arr
def __call__(self, *input_values): # type: (*NumericData) -> NumericData
"""Run computation on input values and return result."""
for tensor_view, value in zip(self.tensor_views, input_values):
if not isinstance(value, np.ndarray):
value = np.array(value)
Computation._write_ndarray_to_tensor_view(value, tensor_view)
result_element_type = self.node.get_element_type()
result_shape = self.node.get_shape()
result_dtype = get_dtype(result_element_type)
result_view = self.runtime.backend.create_tensor(
result_element_type, result_shape)
result_arr = np.empty(result_shape, dtype=result_dtype)
self.backend.call(self.function, [result_view], self.tensor_views)
Computation._read_tensor_view_to_ndarray(result_view, result_arr)
result_arr = result_arr.reshape(result_shape)
return result_arr
def Pad(onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Add padding to the input tensor."""
data = ng_inputs[0]
# Oprator set version 1
paddings = onnx_node.get_attribute_value('paddings')
# Operator set version >= 2
pads = onnx_node.get_attribute_value('pads')
pads = pads if pads is not None else paddings
if pads is None:
raise ValueError('Pad node (s%): pads attribute is required.',
onnx_node.name)
constant = 'constant'
mode = onnx_node.get_attribute_value(
'mode', constant) # 'constant', 'reflect' or 'edge'
value = onnx_node.get_attribute_value('value', 0.)
if len(pads) != 2 * len(data.shape):
raise ValueError(
'Pad node (%s): \'pads rank (%d) should be double of input tensor '
'rank (%d).', onnx_node.name, len(pads), len(data.shape))
# Operator set version 1 accepts only positive values, while operator set version 2 use negative
# values to remove pads elements. Here we check only for latter case.
if any(map(lambda x: x < 0, pads)):
raise NotImplementedError(
'Pad node (%s): removing padding elements is not supported yet.',
onnx_node.name)
if mode != constant:
raise NotImplementedError(
'Pad node (%s): only constant padding is supported.',
onnx_node.name)
# Split paddings into pairs for each axis
pading_below, pading_above = split_pads_into_pairs(pads)
return ng.pad(data,
ng.constant(value, dtype=get_dtype(data.get_element_type())),
pading_below, pading_above)
def __call__(self, *input_values): # type: (*NumericData) -> NumericData
"""Run computation on input values and return result."""
for tensor_view, value in zip(self.tensor_views, input_values):
if not isinstance(value, np.ndarray):
value = np.array(value)
Computation._write_ndarray_to_tensor_view(value, tensor_view)
result_element_type = self.node.get_element_type()
result_shape = self.node.get_shape()
result_dtype = get_dtype(result_element_type)
result_view = self.runtime.backend.make_primary_tensor_view(
result_element_type, result_shape)
result_arr = np.empty(result_shape, dtype=result_dtype)
external = self.runtime.manager.compile(self.function)
call_frame = self.runtime.backend.make_call_frame(external)
call_frame.call([result_view], self.tensor_views)
Computation._read_tensor_view_to_ndarray(result_view, result_arr)
result_arr = result_arr.reshape(result_shape)
return result_arr
def _write_ndarray_to_tensor_view(value: np.ndarray,
tensor_view: Tensor) -> None:
tensor_view_dtype = get_dtype(tensor_view.element_type)
if list(tensor_view.shape) != list(value.shape) and len(
value.shape) > 0:
raise UserInputError(
"Provided tensor's shape: %s does not match the expected: %s.",
list(value.shape),
list(tensor_view.shape),
)
if value.dtype != tensor_view_dtype:
log.warning(
"Attempting to write a %s value to a %s tensor. Will attempt type conversion.",
value.dtype,
tensor_view.element_type,
)
value = value.astype(tensor_view_dtype)
buffer_size = Computation._get_buffer_size(tensor_view.element_type,
tensor_view.element_count)
nparray = np.ascontiguousarray(value)
tensor_view.write(util.numpy_to_c(nparray), buffer_size)
def make_convolution_op(onnx_node, ng_inputs):
# type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""
Create an ngraph convolution Op based on an ONNX node.
:param onnx_node: wrapped ONNX node for Conv of ConvTranspose op
:param ng_inputs: ngraph TensorOp input tensors
:return: ngraph Op for convolution or deconvolution
"""
if len(ng_inputs) == 3:
data, weights, bias = ng_inputs
elif len(ng_inputs) == 2:
data, weights = ng_inputs
bias = ng.constant(0, dtype=get_dtype(data.get_element_type()))
else:
raise ValueError(
'Conv node (%s): unexpected number of input values: %d.',
onnx_node.name, len(ng_inputs))
groups = onnx_node.get_attribute_value('group', 1)
strides = get_strides(onnx_node)
dilation = get_dilations(onnx_node)
padding_below, padding_above = get_pads(onnx_node)
if groups != 1:
# Split one convolution op to N ops where N is the number of groups and concat results after computation.
# reference: https://github.com/NervanaSystems/ngraph-mxnet/blob/fdd692/src/ngraph/ngraph_emitter.cc#L822-L856
data_shape = list(data.shape)
weights_shape = list(weights.shape)
convolutions_nodes = []
# initial bounds for splice
data_lower_part = len(data_shape) * [0]
data_upper_part = copy(data_shape)
weights_lower_part = len(weights_shape) * [0]
weights_upper_part = copy(weights_shape)
for group in range(groups):
# update bounds for splice
data_lower_part[1] = group * int((data_shape[1] / groups))
data_upper_part[1] = (group + 1) * int((data_shape[1] / groups))
sliced_data = ng.slice(data, data_lower_part, data_upper_part)
# update bounds for splice
weights_lower_part[0] = group * int((weights_shape[0] / groups))
weights_upper_part[0] = max((group + 1) * int(
(weights_shape[0] / groups)), 1)
sliced_weights = ng.slice(weights, weights_lower_part,
weights_upper_part)
convolutions_nodes.append(
ng.convolution(sliced_data, sliced_weights, strides, dilation,
padding_below, padding_above))
conv = ng.concat(convolutions_nodes, axis=1)
else:
conv = ng.convolution(data, weights, strides, dilation, padding_below,
padding_above)
if len(bias.shape) > 0:
return conv + ng.broadcast_to(bias, conv.shape, 1)
else:
return conv
def Sum(onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Calculate element-wise sum of the input tensors."""
initial_value_node = ng.constant(
0, get_dtype(ng_inputs[0].get_element_type()))
return reduce(ng.add, ng_inputs, initial_value_node)
def Max(onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Calculate element-wise max of the input tensors."""
np_dtype = get_dtype(ng_inputs[0].get_element_type())
initial_value_node = ng.constant(NumericLimits.min(np_dtype), np_dtype)
return reduce(ng.maximum, ng_inputs, initial_value_node)