def test_simple_tensor_iterator(): X = ov.parameter(Shape([32, 40, 10]), np.float32, "X") Y = ov.parameter(Shape([32, 40, 10]), np.float32, "Y") M = ov.parameter(Shape([32, 2, 10]), np.float32, "M") X_i = ov.parameter(Shape([32, 2, 10]), np.float32, "X_i") Y_i = ov.parameter(Shape([32, 2, 10]), np.float32, "Y_i") M_body = ov.parameter(Shape([32, 2, 10]), np.float32, "M_body") sum = ov.add(X_i, Y_i) Zo = ov.multiply(sum, M_body) body = Model([Zo], [X_i, Y_i, M_body], "body_function") ti = ov.tensor_iterator() ti.set_body(body) ti.set_sliced_input(X_i, X.output(0), 0, 2, 2, 39, 1) ti.set_sliced_input(Y_i, Y.output(0), 0, 2, 2, -1, 1) ti.set_invariant_input(M_body, M.output(0)) out0 = ti.get_iter_value(Zo.output(0), -1) out1 = ti.get_concatenated_slices(Zo.output(0), 0, 2, 2, 39, 1) result0 = ov.result(out0) result1 = ov.result(out1) out0_shape = [32, 2, 10] out1_shape = [32, 40, 10] assert list(result0.get_output_shape(0)) == out0_shape assert list(result1.get_output_shape(0)) == out1_shape assert list(ti.get_output_shape(0)) == out0_shape assert list(ti.get_output_shape(1)) == out1_shape
def test_low_latency2(): X = opset8.parameter(Shape([32, 40, 10]), np.float32, "X") Y = opset8.parameter(Shape([32, 40, 10]), np.float32, "Y") M = opset8.parameter(Shape([32, 2, 10]), np.float32, "M") X_i = opset8.parameter(Shape([32, 2, 10]), np.float32, "X_i") Y_i = opset8.parameter(Shape([32, 2, 10]), np.float32, "Y_i") M_body = opset8.parameter(Shape([32, 2, 10]), np.float32, "M_body") sum = opset8.add(X_i, Y_i) Zo = opset8.multiply(sum, M_body) body = Model([Zo], [X_i, Y_i, M_body], "body_function") ti = opset8.tensor_iterator() ti.set_body(body) ti.set_sliced_input(X_i, X.output(0), 0, 2, 2, 39, 1) ti.set_sliced_input(Y_i, Y.output(0), 0, 2, 2, -1, 1) ti.set_invariant_input(M_body, M.output(0)) out0 = ti.get_iter_value(Zo.output(0), -1) out1 = ti.get_concatenated_slices(Zo.output(0), 0, 2, 2, 39, 1) result0 = opset8.result(out0) result1 = opset8.result(out1) model = Model([result0, result1], [X, Y, M]) m = Manager() m.register_pass(LowLatency2()) m.run_passes(model) # TODO: create TI which will be transformed by LowLatency2 assert count_ops(model, "TensorIterator") == [1]
def simple_if(condition_val): condition = ov.constant(condition_val, dtype=np.bool) # then_body X_t = ov.parameter([2], np.float32, "X") Y_t = ov.parameter([2], np.float32, "Y") then_mul = ov.multiply(X_t, Y_t) then_body_res_1 = ov.result(then_mul) then_body = Model([then_body_res_1], [X_t, Y_t], "then_body_function") # else_body X_e = ov.parameter([2], np.float32, "X") Y_e = ov.parameter([2], np.float32, "Y") add_e = ov.add(X_e, Y_e) else_body_res_1 = ov.result(add_e) else_body = Model([else_body_res_1], [X_e, Y_e], "else_body_function") X = ov.constant([3, 4], dtype=np.float32) Y = ov.constant([2, 1], dtype=np.float32) if_node = ov.if_op(condition) if_node.set_then_body(then_body) if_node.set_else_body(else_body) if_node.set_input(X.output(0), X_t, X_e) if_node.set_input(Y.output(0), Y_t, Y_e) if_res = if_node.set_output(then_body_res_1, else_body_res_1) relu = ov.relu(if_res) return relu
def simple_if(condition_val): condition = ov.constant(condition_val, dtype=np.bool) # then_body X_t = ov.parameter([2], np.float32, "X") Y_t = ov.parameter([2], np.float32, "Y") then_mul = ov.multiply(X_t, Y_t) then_body_res_1 = ov.result(then_mul) then_body = GraphBody([X_t, Y_t], [then_body_res_1]) then_body_inputs = [ TensorIteratorInvariantInputDesc(1, 0), TensorIteratorInvariantInputDesc(2, 1) ] then_body_outputs = [TensorIteratorBodyOutputDesc(0, 0)] # else_body X_e = ov.parameter([2], np.float32, "X") Y_e = ov.parameter([2], np.float32, "Y") add_e = ov.add(X_e, Y_e) else_body_res_1 = ov.result(add_e) else_body = GraphBody([X_e, Y_e], [else_body_res_1]) else_body_inputs = [ TensorIteratorInvariantInputDesc(1, 0), TensorIteratorInvariantInputDesc(2, 1) ] else_body_outputs = [TensorIteratorBodyOutputDesc(0, 0)] X = ov.constant([3, 4], dtype=np.float32) Y = ov.constant([2, 1], dtype=np.float32) if_node = ov.if_op(condition, [X, Y], (then_body, else_body), (then_body_inputs, else_body_inputs), (then_body_outputs, else_body_outputs)) relu = ov.relu(if_node) return relu
def create_function2(shape1=[2, 2], shape2=[2, 2], dtype1=np.float32, dtype2=np.float32): input1 = ops.parameter(shape1, dtype=dtype1, name="input1") input1.get_output_tensor(0).set_names({'input1', 'input1a'}) relu1 = ops.relu(input1) res1 = ops.result(relu1, "res1") res1.get_output_tensor(0).set_names({'res1', 'res1a'}) input2 = ops.parameter(shape2, dtype=dtype2, name="input2") input2.get_output_tensor(0).set_names({'input2', 'input2a'}) relu2 = ops.relu(input2) res2 = ops.result(relu2, "res2") res2.get_output_tensor(0).set_names({'res2', 'res2a'}) function = Model(results=[res1, res2], parameters=[input1, input2], name="TestFunction") return function
def create_simple_if_with_two_outputs(condition_val): condition = ov.constant(condition_val, dtype=np.bool) # then_body X_t = ov.parameter([], np.float32, "X") Y_t = ov.parameter([], np.float32, "Y") Z_t = ov.parameter([], np.float32, "Z") add_t = ov.add(X_t, Y_t) mul_t = ov.multiply(Y_t, Z_t) then_body_res_1 = ov.result(add_t) then_body_res_2 = ov.result(mul_t) then_body = GraphBody([X_t, Y_t, Z_t], [then_body_res_1, then_body_res_2]) then_body_inputs = [ TensorIteratorInvariantInputDesc(1, 0), TensorIteratorInvariantInputDesc(2, 1), TensorIteratorInvariantInputDesc(3, 2) ] then_body_outputs = [ TensorIteratorBodyOutputDesc(0, 0), TensorIteratorBodyOutputDesc(1, 1) ] # else_body X_e = ov.parameter([], np.float32, "X") Z_e = ov.parameter([], np.float32, "Z") W_e = ov.parameter([], np.float32, "W") add_e = ov.add(X_e, W_e) pow_e = ov.power(W_e, Z_e) else_body_res_1 = ov.result(add_e) else_body_res_2 = ov.result(pow_e) else_body = GraphBody([X_e, Z_e, W_e], [else_body_res_1, else_body_res_2]) else_body_inputs = [ TensorIteratorInvariantInputDesc(1, 0), TensorIteratorInvariantInputDesc(3, 1), TensorIteratorInvariantInputDesc(4, 2) ] else_body_outputs = [ TensorIteratorBodyOutputDesc(0, 0), TensorIteratorBodyOutputDesc(1, 1) ] X = ov.constant(15.0, dtype=np.float32) Y = ov.constant(-5.0, dtype=np.float32) Z = ov.constant(4.0, dtype=np.float32) W = ov.constant(2.0, dtype=np.float32) if_node = ov.if_op(condition, [X, Y, Z, W], (then_body, else_body), (then_body_inputs, else_body_inputs), (then_body_outputs, else_body_outputs)) return if_node
def test_simple_loop(): X = ov.parameter(Shape([32, 1, 10]), np.float32, "X") Y = ov.parameter(Shape([32, 1, 10]), np.float32, "Y") M = ov.parameter(Shape([32, 1, 10]), np.float32, "M") input_shape = Shape([]) current_iteration = ov.parameter(Shape([1]), np.int64) X_i = ov.parameter(input_shape, np.float32) Y_i = ov.parameter(input_shape, np.float32) M_body = ov.parameter(input_shape, np.float32) bool_val = np.array([1], dtype=np.bool) bool_val[0] = True body_condition = ov.constant(bool_val) trip_count = ov.constant(10, dtype=np.int64) exec_condition = ov.constant(True, dtype=np.bool) sum = ov.add(X_i, Y_i) Zo = ov.multiply(sum, M_body) body = Model([body_condition, Zo], [current_iteration, X_i, Y_i, M_body], "body_function") loop = ov.loop(trip_count, exec_condition) loop.set_function(body) loop.set_invariant_input(X_i, X.output(0)) loop.set_invariant_input(Y_i, Y.output(0)) loop.set_merged_input(M_body, M.output(0), Zo.output(0)) loop.set_special_body_ports([-1, 0]) out0 = loop.get_iter_value(body_condition.output(0), -1) out1 = loop.get_iter_value(Zo.output(0), -1) out2 = loop.get_concatenated_slices(Zo.output(0), 0, 1, 1, -1, 1) result0 = ov.result(out0) result1 = ov.result(out1) result2 = ov.result(out2) out0_shape = [1] out1_shape = [32, 1, 10] out2_shape = [32, 10, 10] assert list(result0.get_output_shape(0)) == out0_shape assert list(result1.get_output_shape(0)) == out1_shape assert list(result2.get_output_shape(0)) == out2_shape assert list(loop.get_output_shape(0)) == out0_shape assert list(loop.get_output_shape(1)) == out1_shape assert list(loop.get_output_shape(2)) == out2_shape
def test_batched_tensors(device): core = Core() # TODO: remove when plugins will support set_input_tensors core.register_plugin("openvino_template_plugin", "TEMPLATE") batch = 4 one_shape = [1, 2, 2, 2] one_shape_size = np.prod(one_shape) batch_shape = [batch, 2, 2, 2] data1 = ops.parameter(batch_shape, np.float32) data1.set_friendly_name("input0") data1.get_output_tensor(0).set_names({"tensor_input0"}) data1.set_layout(Layout("N...")) constant = ops.constant([1], np.float32) op1 = ops.add(data1, constant) op1.set_friendly_name("Add0") res1 = ops.result(op1) res1.set_friendly_name("Result0") res1.get_output_tensor(0).set_names({"tensor_output0"}) model = Model([res1], [data1]) compiled = core.compile_model(model, "TEMPLATE") req = compiled.create_infer_request() # Allocate 8 chunks, set 'user tensors' to 0, 2, 4, 6 chunks buffer = np.zeros([batch * 2, *batch_shape[1:]], dtype=np.float32) tensors = [] for i in range(batch): # non contiguous memory (i*2) tensors.append( Tensor(np.expand_dims(buffer[i * 2], 0), shared_memory=True)) req.set_input_tensors(tensors) with pytest.raises(RuntimeError) as e: req.get_tensor("tensor_input0") assert "get_tensor shall not be used together with batched set_tensors/set_input_tensors" in str( e.value) actual_tensor = req.get_tensor("tensor_output0") actual = actual_tensor.data for test_num in range(0, 5): for i in range(0, batch): tensors[i].data[:] = test_num + 10 req.infer() # Adds '1' to each element # Reference values for each batch: _tmp = np.array([test_num + 11] * one_shape_size, dtype=np.float32).reshape([2, 2, 2]) for j in range(0, batch): assert np.array_equal(actual[j], _tmp)
def create_model_with_memory(input_shape, data_type): input_data = ops.parameter(input_shape, name="input_data", dtype=data_type) rv = ops.read_value(input_data, "var_id_667") add = ops.add(rv, input_data, name="MemoryAdd") node = ops.assign(add, "var_id_667") res = ops.result(add, "res") model = Model(results=[res], sinks=[node], parameters=[input_data], name="name") return model
def create_function1(shape1=[2, 2]): input1 = ops.parameter(shape1, dtype=np.float32, name="input1") input1.get_output_tensor(0).set_names({'input1a', 'input1b'}) relu1 = ops.relu(input1) res1 = ops.result(relu1, "res1") res1.get_output_tensor(0).set_names({'res1', 'res1a'}) function = Model(results=[res1], parameters=[input1], name="TestFunction") return function
def get_model(): param = opset8.parameter(PartialShape([1, 3, 22, 22]), name="parameter") param.get_output_tensor(0).set_names({"parameter"}) relu = opset8.relu(param) reshape = opset8.reshape(relu, opset8.shape_of(relu), False) res = opset8.result(reshape, name="result") res.get_output_tensor(0).set_names({"result"}) return Model([res], [param], "test")
def test_batched_tensors(device): batch = 4 one_shape = Shape([1, 2, 2, 2]) batch_shape = Shape([batch, 2, 2, 2]) one_shape_size = np.prod(one_shape) core = Core() core.register_plugin("openvino_template_plugin", "TEMPLATE") data1 = ops.parameter(batch_shape, np.float32) data1.set_friendly_name("input0") data1.get_output_tensor(0).set_names({"tensor_input0"}) data1.set_layout(Layout("N...")) constant = ops.constant([1], np.float32) op1 = ops.add(data1, constant) op1.set_friendly_name("Add0") res1 = ops.result(op1) res1.set_friendly_name("Result0") res1.get_output_tensor(0).set_names({"tensor_output0"}) model = Model([res1], [data1]) compiled = core.compile_model(model, "TEMPLATE") buffer = np.zeros([one_shape_size * batch * 2], dtype=np.float32) req = compiled.create_infer_request() tensors = [] for i in range(0, batch): _start = i * one_shape_size * 2 # Use of special constructor for Tensor. # It creates a Tensor from pointer, thus it requires only # one element from original buffer, and shape to "crop". tensor = Tensor(buffer[_start:(_start + 1)], one_shape) tensors.append(tensor) req.set_input_tensors(tensors) # using list overload! actual_tensor = req.get_tensor("tensor_output0") actual = actual_tensor.data for test_num in range(0, 5): for i in range(0, batch): tensors[i].data[:] = test_num + 10 req.infer() # Adds '1' to each element # Reference values for each batch: _tmp = np.array([test_num + 11] * one_shape_size, dtype=np.float32).reshape([2, 2, 2]) for j in range(0, batch): assert np.array_equal(actual[j], _tmp)
def test_replace_output_update_name(): param = opset8.parameter(PartialShape([1, 3, 22, 22]), name="parameter") relu = opset8.relu(param.output(0)) exp = opset8.exp(relu.output(0)) res = opset8.result(exp.output(0), name="result") replace_output_update_name(exp.output(0), exp.input_value(0)) assert res.input_value(0).get_node() == exp
def create_diff_if_with_two_outputs(condition_val): condition = ov.constant(condition_val, dtype=np.bool) # then_body X_t = ov.parameter([2], np.float32, "X") Y_t = ov.parameter([2], np.float32, "Y") mmul_t = ov.matmul(X_t, Y_t, False, False) mul_t = ov.multiply(Y_t, X_t) then_body_res_1 = ov.result(mmul_t) then_body_res_2 = ov.result(mul_t) then_body = GraphBody([X_t, Y_t], [then_body_res_1, then_body_res_2]) then_body_inputs = [ TensorIteratorInvariantInputDesc(1, 0), TensorIteratorInvariantInputDesc(2, 1) ] then_body_outputs = [ TensorIteratorBodyOutputDesc(0, 0), TensorIteratorBodyOutputDesc(1, 1) ] # else_body X_e = ov.parameter([2], np.float32, "X") Z_e = ov.parameter([], np.float32, "Z") mul_e = ov.multiply(X_e, Z_e) else_body_res_1 = ov.result(Z_e) else_body_res_2 = ov.result(mul_e) else_body = GraphBody([X_e, Z_e], [else_body_res_1, else_body_res_2]) else_body_inputs = [ TensorIteratorInvariantInputDesc(1, 0), TensorIteratorInvariantInputDesc(3, 1) ] else_body_outputs = [ TensorIteratorBodyOutputDesc(0, 0), TensorIteratorBodyOutputDesc(1, 1) ] X = ov.constant([3, 4], dtype=np.float32) Y = ov.constant([2, 1], dtype=np.float32) Z = ov.constant(4.0, dtype=np.float32) if_node = ov.if_op(condition, [X, Y, Z], (then_body, else_body), (then_body_inputs, else_body_inputs), (then_body_outputs, else_body_outputs)) return if_node
def simple_if_without_parameters(condition_val): condition = ov.constant(condition_val, dtype=np.bool) # then_body then_constant = ov.constant(0.7, dtype=np.float) then_body_res_1 = ov.result(then_constant) then_body = Model([then_body_res_1], []) # else_body else_const = ov.constant(9.0, dtype=np.float) else_body_res_1 = ov.result(else_const) else_body = Model([else_body_res_1], []) if_node = ov.if_op(condition.output(0)) if_node.set_then_body(then_body) if_node.set_else_body(else_body) if_res = if_node.set_output(then_body_res_1, else_body_res_1) relu = ov.relu(if_res) return relu
def create_simple_if_with_two_outputs(condition_val): condition = ov.constant(condition_val, dtype=np.bool) # then_body X_t = ov.parameter([], np.float32, "X") Y_t = ov.parameter([], np.float32, "Y") Z_t = ov.parameter([], np.float32, "Z") add_t = ov.add(X_t, Y_t) mul_t = ov.multiply(Y_t, Z_t) then_body_res_1 = ov.result(add_t) then_body_res_2 = ov.result(mul_t) then_body = Model([then_body_res_1, then_body_res_2], [X_t, Y_t, Z_t], "then_body_function") # else_body X_e = ov.parameter([], np.float32, "X") Z_e = ov.parameter([], np.float32, "Z") W_e = ov.parameter([], np.float32, "W") add_e = ov.add(X_e, W_e) pow_e = ov.power(W_e, Z_e) else_body_res_1 = ov.result(add_e) else_body_res_2 = ov.result(pow_e) else_body = Model([else_body_res_1, else_body_res_2], [X_e, Z_e, W_e], "else_body_function") X = ov.constant(15.0, dtype=np.float32) Y = ov.constant(-5.0, dtype=np.float32) Z = ov.constant(4.0, dtype=np.float32) W = ov.constant(2.0, dtype=np.float32) if_node = ov.if_op(condition) if_node.set_then_body(then_body) if_node.set_else_body(else_body) if_node.set_input(X.output(0), X_t, X_e) if_node.set_input(Y.output(0), Y_t, None) if_node.set_input(Z.output(0), Z_t, Z_e) if_node.set_input(W.output(0), None, W_e) if_node.set_output(then_body_res_1, else_body_res_1) if_node.set_output(then_body_res_2, else_body_res_2) return if_node
def test_replace_source_output(): param = opset8.parameter(PartialShape([1, 3, 22, 22]), name="parameter") relu = opset8.relu(param.output(0)) res = opset8.result(relu.output(0), name="result") exp = opset8.exp(param.output(0)) res.input(0).replace_source_output(exp.output(0)) assert len(exp.output(0).get_target_inputs()) == 1 assert len(relu.output(0).get_target_inputs()) == 0 assert next(iter(exp.output(0).get_target_inputs())).get_node() == res
def simple_if_without_parameters(condition_val): condition = ov.constant(condition_val, dtype=np.bool) # then_body then_constant = ov.constant(0.7, dtype=np.float) then_body_res_1 = ov.result(then_constant) then_body = GraphBody([], [then_body_res_1]) then_body_inputs = [] then_body_outputs = [TensorIteratorBodyOutputDesc(0, 0)] # else_body else_const = ov.constant(9.0, dtype=np.float) else_body_res_1 = ov.result(else_const) else_body = GraphBody([], [else_body_res_1]) else_body_inputs = [] else_body_outputs = [TensorIteratorBodyOutputDesc(0, 0)] if_node = ov.if_op(condition, [], (then_body, else_body), (then_body_inputs, else_body_inputs), (then_body_outputs, else_body_outputs)) relu = ov.relu(if_node) return relu
def create_diff_if_with_two_outputs(condition_val): condition = ov.constant(condition_val, dtype=np.bool) # then_body X_t = ov.parameter([2], np.float32, "X") Y_t = ov.parameter([2], np.float32, "Y") mmul_t = ov.matmul(X_t, Y_t, False, False) mul_t = ov.multiply(Y_t, X_t) then_body_res_1 = ov.result(mmul_t) then_body_res_2 = ov.result(mul_t) then_body = Model([then_body_res_1, then_body_res_2], [X_t, Y_t], "then_body_function") # else_body X_e = ov.parameter([2], np.float32, "X") Z_e = ov.parameter([], np.float32, "Z") mul_e = ov.multiply(X_e, Z_e) else_body_res_1 = ov.result(Z_e) else_body_res_2 = ov.result(mul_e) else_body = Model([else_body_res_1, else_body_res_2], [X_e, Z_e], "else_body_function") X = ov.constant([3, 4], dtype=np.float32) Y = ov.constant([2, 1], dtype=np.float32) Z = ov.constant(4.0, dtype=np.float32) if_node = ov.if_op(condition) if_node.set_then_body(then_body) if_node.set_else_body(else_body) if_node.set_input(X.output(0), X_t, X_e) if_node.set_input(Y.output(0), Y_t, None) if_node.set_input(Z.output(0), None, Z_e) if_node.set_output(then_body_res_1, else_body_res_1) if_node.set_output(then_body_res_2, else_body_res_2) return if_node
def test_simple_if_basic(): condition = ov.constant(True, dtype=np.bool) # then_body X_t = ov.parameter([2], np.float32, "X") Y_t = ov.parameter([2], np.float32, "Y") then_mul = ov.multiply(X_t, Y_t) then_body_res_1 = ov.result(then_mul) then_body = Model([then_body_res_1], [X_t, Y_t], "then_body_function") then_body_inputs = [ InvariantInputDescription(1, 0), InvariantInputDescription(2, 1) ] else_body_outputs = [BodyOutputDescription(0, 0)] if_node = ov.if_op(condition.output(0)) if_node.set_function(0, then_body) assert if_node.get_function(0) == then_body if_node.set_input_descriptions(0, then_body_inputs) if_node.set_output_descriptions(1, else_body_outputs) input_desc = if_node.get_input_descriptions(0) output_desc = if_node.get_output_descriptions(1) assert len(input_desc) == len(then_body_inputs) assert len(output_desc) == len(else_body_outputs) for i in range(len(input_desc)): assert input_desc[i].get_type_info( ) == then_body_inputs[i].get_type_info() assert input_desc[i].input_index == then_body_inputs[i].input_index assert input_desc[i].body_parameter_index == then_body_inputs[ i].body_parameter_index for i in range(len(output_desc)): assert output_desc[i].get_type_info( ) == else_body_outputs[i].get_type_info() assert output_desc[i].output_index == else_body_outputs[i].output_index assert output_desc[i].body_value_index == else_body_outputs[ i].body_value_index
def test_sink_function_ctor(): input_data = ops.parameter([2, 2], name="input_data", dtype=np.float32) rv = ops.read_value(input_data, "var_id_667") add = ops.add(rv, input_data, name="MemoryAdd") node = ops.assign(add, "var_id_667") res = ops.result(add, "res") function = Model(results=[res], sinks=[node], parameters=[input_data], name="TestFunction") ordered_ops = function.get_ordered_ops() op_types = [op.get_type_name() for op in ordered_ops] assert op_types == ["Parameter", "ReadValue", "Add", "Assign", "Result"] assert len(function.get_ops()) == 5 assert function.get_output_size() == 1 assert function.get_output_op(0).get_type_name() == "Result" assert function.get_output_element_type(0) == input_data.get_element_type() assert list(function.get_output_shape(0)) == [2, 2] assert (function.get_parameters()[0].get_partial_shape()) == PartialShape( [2, 2]) assert len(function.get_parameters()) == 1 assert len(function.get_results()) == 1 assert function.get_friendly_name() == "TestFunction"
def get_test_function(): # Parameter->Relu->Result param = opset8.parameter(PartialShape([1, 3, 22, 22]), name="parameter") relu = opset8.relu(param.output(0)) res = opset8.result(relu.output(0), name="result") return Model([res], [param], "test")