def Mean(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Calculate element-wise mean of the input tensors.""" initial_value_node = ng.constant( 0, get_dtype(ng_inputs[0].get_element_type())) sum_node = reduce(ng.add, ng_inputs, initial_value_node) count_array = np.full(sum_node.shape, len(ng_inputs), dtype=get_dtype(sum_node.get_element_type())) return sum_node / ng.constant(count_array)
def make_convolution_op(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """ Create an ngraph convolution Op based on an ONNX node. :param onnx_node: wrapped ONNX node for Conv of ConvTranspose op :param ng_inputs: ngraph TensorOp input tensors :return: ngraph Op for convolution or deconvolution """ if len(ng_inputs) == 3: x, weights, bias = ng_inputs elif len(ng_inputs) == 2: x, weights = ng_inputs bias = ng.constant(0, dtype=get_dtype(x.get_element_type())) else: raise ValueError( 'Conv node (%s): unexpected number of input values: %d.', onnx_node.name, len(ng_inputs)) groups = onnx_node.get_attribute_value('group', 1) if groups != 1: log.warning( 'Conv node (%s): `group` attribute value %d is not supported.', onnx_node.name, groups) strides = get_strides(onnx_node) dilation = get_dilations(onnx_node) padding_below, padding_above = get_pads(onnx_node) conv = ng.convolution(x, weights, strides, dilation, padding_below, padding_above) return conv + bias
def as_elementwise_compatible_nodes(*input_values): # type: (*NodeInput) -> List[Node] """Return all input values as ngraph Nodes with the same shape and element type. Scalar values will be converted to ngraph Constant Nodes. """ input_nodes = [node for node in input_values if issubclass(type(node), Node)] # type: List[Node] if not input_nodes: raise NotImplementedError('Operations on scalars only are not supported.') shapes = {tuple(node.shape) for node in input_nodes} if len(shapes) > 1: log.warning('More than one different shape in input nodes %s.', input_nodes) types = [node.get_element_type() for node in input_nodes] unique_types = {repr(type) for type in types} if len(unique_types) > 1: log.warning('More than one different data type in input nodes %s.', input_nodes) sorted_shapes = sorted(shapes, key=len) broadcast_shape = sorted_shapes.pop() broadcast_dtype = get_dtype(types.pop()) output_nodes = [] for input_value in input_values: if issubclass(type(input_value), Node): input_value = ng.broadcast(input_value, broadcast_shape) output_nodes.append(input_value) else: input_value = make_constant_node(input_value, dtype=broadcast_dtype) output_nodes.append(ng.broadcast(input_value, broadcast_shape)) return output_nodes
def BatchNormalization(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Carry out batch normalization.""" x, scale, bias, mean, var = ng_inputs is_test = onnx_node.get_attribute_value('is_test', 1) spatial = onnx_node.get_attribute_value('spatial', 1) epsilon = onnx_node.get_attribute_value('epsilon', 1e-3) # @TODO: Implement learning mode support # momentum = onnx_node.get_attribute_value('momentum', 0.99) if not is_test: raise NotImplementedError('BatchNormalization node (%s): only `is_test` mode is currently ' 'supported.', onnx_node.name) if not spatial: raise NotImplementedError('BatchNormalization node (%s): only `spatial` mode is currently ' 'supported.', onnx_node.name) mean = ng.broadcast(mean, x.shape, axis=1) scale = ng.broadcast(scale, x.shape, axis=1) var = ng.broadcast(var, x.shape, axis=1) bias = ng.broadcast(bias, x.shape, axis=1) epsilon = ng.broadcast(ng.constant(epsilon, dtype=get_dtype(x.get_element_type())), x.shape, axis=1) return (scale * ((x - mean) * (1 / (ng.sqrt(var + epsilon)))) + bias)
def Cast(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Limit input tensor values within specified interval.""" data = ng_inputs[0] cast_to_type = onnx_node.get_attribute_value('to') if cast_to_type is None: raise ValueError('Cast node (%s): \'to\' attribute is required.') input_tensor_type = get_dtype(data.get_element_type()) new_type = onnx_tensor_type_to_numpy_type(cast_to_type) unsupported_types = [ onnx_tensor_type_to_numpy_type('COMPLEX64'), onnx_tensor_type_to_numpy_type('COMPLEX128'), ] if input_tensor_type in unsupported_types: raise ValueError( 'Cast node (%s): input tensor data type (%s) is not supported.', onnx_node.name, str(input_tensor_type)) if new_type in unsupported_types: raise ValueError( 'Cast node (%s): casting to type (%s) is not supported.', onnx_node.name, str(new_type)) return ng.convert(data, new_type)
def as_elementwise_compatible_nodes(*input_values): # type: (*NodeInput) -> List[Node] """Return all input values as ngraph Nodes with the same shape and element type. Scalar values will be converted to ngraph Constant Nodes. """ input_nodes = [node for node in input_values if issubclass(type(node), Node)] # type: List[Node] if not input_nodes: raise NotImplementedError('Operations on scalars only are not supported.') shapes = {tuple(node.shape) for node in input_nodes} if len(shapes) > 1: log.warning('More than one different shape in input nodes %s.', input_nodes) types = [node.get_element_type() for node in input_nodes] unique_types = {repr(type) for type in types} if len(unique_types) > 1: log.warning('More than one different data type in input nodes %s.', input_nodes) sorted_shapes = sorted(shapes, key=len) broadcast_shape = sorted_shapes.pop() broadcast_dtype = get_dtype(types.pop()) output_nodes = [] for input_value in input_values: if issubclass(type(input_value), Node): input_value = ng.broadcast_to(input_value, broadcast_shape) output_nodes.append(input_value) else: input_value = make_constant_node(input_value, dtype=broadcast_dtype) output_nodes.append(ng.broadcast_to(input_value, broadcast_shape)) return output_nodes
def ReduceMean(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Compute the mean value of the input tensor's elements along the provided axes.""" input_shape = list(ng_inputs[0].shape) sum_node = make_reduction_op(ng.sum, onnx_node, ng_inputs[0]) reduction_axes = get_reduction_axes(onnx_node, ng_inputs[0]) avg_elem_count = np.prod([input_shape[x] for x in reduction_axes]) const_node = ng.broadcast(ng.constant(avg_elem_count, get_dtype(sum_node.get_element_type())), sum_node.shape) return ng.divide(sum_node, const_node)
def ConvTranspose( onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Calculate convolution transpose.""" if len(ng_inputs) == 3: data, weights, bias = ng_inputs elif len(ng_inputs) == 2: data, weights = ng_inputs bias = ng.constant(0, dtype=get_dtype(data.get_element_type())) strides = get_strides(onnx_node) dilation = get_dilations(onnx_node) padding_below, padding_above = get_pads(onnx_node) output_padding = onnx_node.get_attribute_value('output_padding') if output_padding is None: raise ValueError( 'ConvTranspose node (s%): output_padding attribute is required.', onnx_node.name) data_shape = list(data.shape) weights_shape = list(weights.shape) num_spatial_dims = len(data.shape) - 2 data_dilation_strides = [1, 1] data_batch_shape = [1] * (num_spatial_dims + 2) data_batch_shape[0] = data_shape[0] data_batch_shape[1] = weights_shape[1] for i in range(num_spatial_dims): # Calculating spatial dims of data output shape for ngraph conv backprop op # | pb + s(ds-1) + op - d(ws-1)+1 | # | ----------------------------- | + 1 # |_ dds _| # # d - dilation # ds - data shape # dds - data dilation strides # op - putput padding # pb - padding below # s - strides # ws - weights shape data_batch_shape[i + 2] = ( (padding_below[i] + ((data_shape[i + 2] - 1) * strides[i] + 1) + output_padding[i]) - ((weights_shape[i + 2] - 1) * dilation[i] + 1) + 1) // data_dilation_strides[i] + 1 transconv = ng.convolution_backprop_data(data_batch_shape, weights, data, strides, dilation, padding_below, padding_above, data_dilation_strides) if len(bias.shape) > 0: return transconv + ng.broadcast_to(bias, transconv.shape, 1) else: return transconv
def __call__(self, *input_values: NumericData) -> List[NumericData]: """Run computation on input values and return result.""" input_values = [np.array(input_value) for input_value in input_values] input_shapes = [get_shape(input_value) for input_value in input_values] param_names = [param.friendly_name for param in self.parameters] if self.network_cache.get(str(input_shapes)) is None: capsule = Function.to_capsule(self.function) cnn_network = IENetwork(capsule) if self.function.is_dynamic(): cnn_network.reshape(dict(zip(param_names, input_shapes))) # Convert unsupported inputs of the network _convert_inputs(cnn_network) self.network_cache[str(input_shapes)] = cnn_network else: cnn_network = self.network_cache[str(input_shapes)] executable_network = self.runtime.backend.load_network( cnn_network, self.runtime.backend_name) # Input validation if len(input_values) != len(self.parameters): raise UserInputError("Expected %s parameters, received %s.", len(self.parameters), len(input_values)) for parameter, input in zip(self.parameters, input_values): parameter_shape = parameter.get_output_partial_shape(0) input_shape = PartialShape(input.shape) if len(input.shape) > 0 and not parameter_shape.compatible( input_shape): raise UserInputError( "Provided tensor's shape: %s does not match the expected: %s.", input_shape, parameter_shape, ) request = executable_network.requests[0] request.infer(dict(zip(param_names, input_values))) # Set order of output blobs compatible with nG Function result_buffers = [ self.__get_ie_output_blob_buffer(request.output_blobs, result) for result in self.results ] # Since OV overwrite result data type we have to convert results to the original one. original_dtypes = [ get_dtype(result.get_output_element_type(0)) for result in self.results ] converted_buffers = [ buffer.astype(original_dtype) for buffer, original_dtype in zip(result_buffers, original_dtypes) ] return converted_buffers
def ThresholdedRelu( onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Apply the Thresholded Relu function to the input tensor elementwise. f(x) = 0 for x <= alpha, f(x) = x for x > alpha """ x = ng_inputs[0] alpha = onnx_node.get_attribute_value('alpha', 1.0) x_map = ng.convert(ng.greater(x, alpha), get_dtype(x.get_element_type())) x = x * x_map return x
def Clip(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Limit input tensor values within specified interval.""" data = ng_inputs[0] data_elem_dtype = get_dtype(data.get_element_type()) max_value = onnx_node.get_attribute_value('max', np.finfo(data_elem_dtype).max) min_value = onnx_node.get_attribute_value('min', np.finfo(data_elem_dtype).min) return ng.minimum( ng.maximum(data, ng.constant(min_value, data_elem_dtype)), ng.constant(max_value, data_elem_dtype))
def _write_ndarray_to_tensor_view(value, tensor_view): # type: (np.ndarray, TensorViewType) -> None tensor_view_dtype = get_dtype(tensor_view.element_type) if value.dtype != tensor_view_dtype: log.warning( 'Attempting to write a %s value to a %s tensor. Will attempt type conversion.', value.dtype, tensor_view.element_type) value = value.astype(tensor_view_dtype) buffer_size = Computation._get_buffer_size( tensor_view.element_type, tensor_view.element_count) tensor_view.write(util.numpy_to_c(np.ascontiguousarray(value)), 0, buffer_size)
def HardSigmoid( onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Apply f(x) = max(0, min(1, alpha * x + beta)) function to tensor element-wise. :param onnx_node: The ONNX node representing this operation. :param ng_inputs: The input tensors. :return: The tensor with applied HardSigmoid operation. """ data = ng_inputs[0] data_type = get_dtype(data.get_element_type()).type alpha = onnx_node.get_attribute_value('alpha', float(0.2)) beta = onnx_node.get_attribute_value('beta', float(0.5)) return ng.maximum(data_type(0), ng.minimum(data_type(1), alpha * data + beta))
def __call__(self, *input_values): # type: (*NumericData) -> List[NumericData] """Run computation on input values and return result.""" for tensor_view, value in zip(self.tensor_views, input_values): if not isinstance(value, np.ndarray): value = np.array(value) Computation._write_ndarray_to_tensor_view(value, tensor_view) self.handle.call(self.result_views, self.tensor_views) results = [] for result_view in self.result_views: result = np.ndarray(result_view.shape, dtype=get_dtype(result_view.element_type)) Computation._read_tensor_view_to_ndarray(result_view, result) results.append(result) return results
def get_bool_nodes( nodes): # type: (Tuple[NgraphNode, ...]) -> Tuple[NgraphNode, ...] """Convert each input node to bool data type if necessary. :param nodes: Input nodes to be converted. :return: Converted nodes. """ bool_nodes = [] for node in nodes: if not node.get_element_type() == NgraphType.boolean: bool_nodes.append(ng.convert(node, bool)) logger.warning('Converting node of type: <{}> to bool.'.format( get_dtype(node.get_element_type()))) else: bool_nodes.append(node) return tuple(bool_nodes)
def _write_ndarray_to_tensor_view(value, tensor_view): # type: (np.ndarray, TensorViewType) -> None tensor_view_dtype = get_dtype(tensor_view.element_type) if list(tensor_view.shape) != list(value.shape) and len( value.shape) > 0: raise UserInputError( 'Provided tensor\'s shape: %s does not match the expected: %s.', list(value.shape), list(tensor_view.shape)) if value.dtype != tensor_view_dtype: log.warning( 'Attempting to write a %s value to a %s tensor. Will attempt type conversion.', value.dtype, tensor_view.element_type) value = value.astype(tensor_view_dtype) buffer_size = Computation._get_buffer_size(tensor_view.element_type, tensor_view.element_count) tensor_view.write(util.numpy_to_c(np.ascontiguousarray(value)), 0, buffer_size)
def __call__(self, *input_values): # type: (*NumericData) -> NumericData """Run computation on input values and return result.""" for tensor_view, value in zip(self.tensor_views, input_values): if not isinstance(value, np.ndarray): value = np.array(value) Computation._write_ndarray_to_tensor_view(value, tensor_view) result_element_type = self.function.get_output_element_type(0) result_shape = self.function.get_output_shape(0) result_dtype = get_dtype(result_element_type) result_view = self.runtime.backend.create_tensor(result_element_type, result_shape) result_arr = np.empty(result_shape, dtype=result_dtype) self.runtime.backend.call(self.function, [result_view], self.tensor_views) Computation._read_tensor_view_to_ndarray(result_view, result_arr) result_arr = result_arr.reshape(result_shape) return result_arr
def __call__(self, *input_values): # type: (*NumericData) -> NumericData """Run computation on input values and return result.""" for tensor_view, value in zip(self.tensor_views, input_values): if not isinstance(value, np.ndarray): value = np.array(value) Computation._write_ndarray_to_tensor_view(value, tensor_view) result_element_type = self.node.get_element_type() result_shape = self.node.get_shape() result_dtype = get_dtype(result_element_type) result_view = self.runtime.backend.create_tensor( result_element_type, result_shape) result_arr = np.empty(result_shape, dtype=result_dtype) self.backend.call(self.function, [result_view], self.tensor_views) Computation._read_tensor_view_to_ndarray(result_view, result_arr) result_arr = result_arr.reshape(result_shape) return result_arr
def Pad(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Add padding to the input tensor.""" data = ng_inputs[0] # Oprator set version 1 paddings = onnx_node.get_attribute_value('paddings') # Operator set version >= 2 pads = onnx_node.get_attribute_value('pads') pads = pads if pads is not None else paddings if pads is None: raise ValueError('Pad node (s%): pads attribute is required.', onnx_node.name) constant = 'constant' mode = onnx_node.get_attribute_value( 'mode', constant) # 'constant', 'reflect' or 'edge' value = onnx_node.get_attribute_value('value', 0.) if len(pads) != 2 * len(data.shape): raise ValueError( 'Pad node (%s): \'pads rank (%d) should be double of input tensor ' 'rank (%d).', onnx_node.name, len(pads), len(data.shape)) # Operator set version 1 accepts only positive values, while operator set version 2 use negative # values to remove pads elements. Here we check only for latter case. if any(map(lambda x: x < 0, pads)): raise NotImplementedError( 'Pad node (%s): removing padding elements is not supported yet.', onnx_node.name) if mode != constant: raise NotImplementedError( 'Pad node (%s): only constant padding is supported.', onnx_node.name) # Split paddings into pairs for each axis pading_below, pading_above = split_pads_into_pairs(pads) return ng.pad(data, ng.constant(value, dtype=get_dtype(data.get_element_type())), pading_below, pading_above)
def __call__(self, *input_values): # type: (*NumericData) -> NumericData """Run computation on input values and return result.""" for tensor_view, value in zip(self.tensor_views, input_values): if not isinstance(value, np.ndarray): value = np.array(value) Computation._write_ndarray_to_tensor_view(value, tensor_view) result_element_type = self.node.get_element_type() result_shape = self.node.get_shape() result_dtype = get_dtype(result_element_type) result_view = self.runtime.backend.make_primary_tensor_view( result_element_type, result_shape) result_arr = np.empty(result_shape, dtype=result_dtype) external = self.runtime.manager.compile(self.function) call_frame = self.runtime.backend.make_call_frame(external) call_frame.call([result_view], self.tensor_views) Computation._read_tensor_view_to_ndarray(result_view, result_arr) result_arr = result_arr.reshape(result_shape) return result_arr
def _write_ndarray_to_tensor_view(value: np.ndarray, tensor_view: Tensor) -> None: tensor_view_dtype = get_dtype(tensor_view.element_type) if list(tensor_view.shape) != list(value.shape) and len( value.shape) > 0: raise UserInputError( "Provided tensor's shape: %s does not match the expected: %s.", list(value.shape), list(tensor_view.shape), ) if value.dtype != tensor_view_dtype: log.warning( "Attempting to write a %s value to a %s tensor. Will attempt type conversion.", value.dtype, tensor_view.element_type, ) value = value.astype(tensor_view_dtype) buffer_size = Computation._get_buffer_size(tensor_view.element_type, tensor_view.element_count) nparray = np.ascontiguousarray(value) tensor_view.write(util.numpy_to_c(nparray), buffer_size)
def make_convolution_op(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """ Create an ngraph convolution Op based on an ONNX node. :param onnx_node: wrapped ONNX node for Conv of ConvTranspose op :param ng_inputs: ngraph TensorOp input tensors :return: ngraph Op for convolution or deconvolution """ if len(ng_inputs) == 3: data, weights, bias = ng_inputs elif len(ng_inputs) == 2: data, weights = ng_inputs bias = ng.constant(0, dtype=get_dtype(data.get_element_type())) else: raise ValueError( 'Conv node (%s): unexpected number of input values: %d.', onnx_node.name, len(ng_inputs)) groups = onnx_node.get_attribute_value('group', 1) strides = get_strides(onnx_node) dilation = get_dilations(onnx_node) padding_below, padding_above = get_pads(onnx_node) if groups != 1: # Split one convolution op to N ops where N is the number of groups and concat results after computation. # reference: https://github.com/NervanaSystems/ngraph-mxnet/blob/fdd692/src/ngraph/ngraph_emitter.cc#L822-L856 data_shape = list(data.shape) weights_shape = list(weights.shape) convolutions_nodes = [] # initial bounds for splice data_lower_part = len(data_shape) * [0] data_upper_part = copy(data_shape) weights_lower_part = len(weights_shape) * [0] weights_upper_part = copy(weights_shape) for group in range(groups): # update bounds for splice data_lower_part[1] = group * int((data_shape[1] / groups)) data_upper_part[1] = (group + 1) * int((data_shape[1] / groups)) sliced_data = ng.slice(data, data_lower_part, data_upper_part) # update bounds for splice weights_lower_part[0] = group * int((weights_shape[0] / groups)) weights_upper_part[0] = max((group + 1) * int( (weights_shape[0] / groups)), 1) sliced_weights = ng.slice(weights, weights_lower_part, weights_upper_part) convolutions_nodes.append( ng.convolution(sliced_data, sliced_weights, strides, dilation, padding_below, padding_above)) conv = ng.concat(convolutions_nodes, axis=1) else: conv = ng.convolution(data, weights, strides, dilation, padding_below, padding_above) if len(bias.shape) > 0: return conv + ng.broadcast_to(bias, conv.shape, 1) else: return conv
def Sum(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Calculate element-wise sum of the input tensors.""" initial_value_node = ng.constant( 0, get_dtype(ng_inputs[0].get_element_type())) return reduce(ng.add, ng_inputs, initial_value_node)
def Max(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Calculate element-wise max of the input tensors.""" np_dtype = get_dtype(ng_inputs[0].get_element_type()) initial_value_node = ng.constant(NumericLimits.min(np_dtype), np_dtype) return reduce(ng.maximum, ng_inputs, initial_value_node)