def execute(self):
"""Execyte Inference only demo"""
# Create InferServer
cnis.bind_device(self.dev_id)
infer_server = cnis.InferServer(self.dev_id)
# Create session. Sync API
session_desc = cnis.SessionDesc()
session_desc.name = "test_session_sync"
session_desc.engine_num = 1
session_desc.strategy = cnis.BatchStrategy.STATIC
session_desc.model = infer_server.load_model(self.model_dir)
session = infer_server.create_sync_session(session_desc)
width, height = self.get_model_input_wh(session_desc.model)
for _ in range(4):
# Prepare input and output
input_pak = self.prepare_input_and_preproc(width, height)
output_pak = cnis.Package(1)
# Request
status = cnis.Status.SUCCESS
ret = infer_server.request_sync(session, input_pak, status, output_pak, timeout=20000)
if not ret:
print("[EasyDK PythonAPISamples] [InferOnlyDemo] RequestSync failed, ret: {}, status: {}".format(
ret, status))
# Postproc and print results
if status == cnis.Status.SUCCESS:
self.postproc_and_print(output_pak, session_desc.model, 0.6)
# Destroy Session
infer_server.destroy_session(session)
def test_python_postprocess():
"""Test custom postprocess function (written in python)"""
# Create InferServer
infer_server = cnis.InferServer(dev_id=0)
session_desc = cnis.SessionDesc()
session_desc.name = "test_session"
# Load model
session_desc.model = infer_server.load_model(utils.model_dir)
# Create Postprocessor and set custom postprocess function to description
session_desc.postproc = cnis.Postprocessor()
custom_python_postproc = CustomPostprocess()
session_desc.set_postproc_func(custom_python_postproc.execute)
session_desc.show_perf = False
# Create synchronous session
session = infer_server.create_sync_session(session_desc)
request_package(session_desc, infer_server, session)
# Check if custom postprocess function is called
assert custom_python_postproc.called
# Destroy session
assert infer_server.destroy_session(session)
def test_opencv_frame():
# Create InferServer
infer_server = cnis.InferServer(dev_id=0)
session_desc = cnis.SessionDesc()
session_desc.name = "test_session"
session_desc.model = infer_server.load_model(utils.model_dir)
# Set OpencvPreproc
session_desc.preproc = cnis.PreprocessorHost()
session_desc.set_preproc_func(cnis.OpencvPreproc(dst_fmt=cnis.VideoPixelFmt.BGRA, keep_aspect_ratio=False).execute)
session_desc.show_perf = False
# create synchronous session
session = infer_server.create_sync_session(session_desc)
# Request opencv frame
input_pak = prepare_input()
output = cnis.Package(1, utils.tag)
status = cnis.Status.SUCCESS
assert infer_server.request_sync(session, input_pak, status, output)
assert status == cnis.Status.SUCCESS
# Request opencv frame with roi
input_pak = prepare_input()
cv_frame = input_pak.data[0].get_cv_frame()
cv_frame.roi = cnis.BoundingBox(x=0, y=0, w=0.5, h=0.5)
output = cnis.Package(1, utils.tag)
status = cnis.Status.SUCCESS
assert infer_server.request_sync(session, input_pak, status, output)
assert status == cnis.Status.SUCCESS
# Destroy session
assert infer_server.destroy_session(session)
def test_python_preprocess():
"""Test custom preprocess function (written in python)"""
# Create InferServer
infer_server = cnis.InferServer(dev_id=0)
session_desc = cnis.SessionDesc()
session_desc.name = "test_session"
# Load model
session_desc.model = infer_server.load_model(utils.model_dir)
# Create PreprocessorHost and set custom preprocess function to description
session_desc.preproc = cnis.PreprocessorHost()
custom_python_preproc = CustomPreprocess()
session_desc.set_preproc_func(custom_python_preproc.execute)
session_desc.show_perf = False
# Create synchronous session
session = infer_server.create_sync_session(session_desc)
# Create a package
input_pak = cnis.Package(1, utils.tag)
output = cnis.Package(1, utils.tag)
status = cnis.Status.SUCCESS
# Request
assert infer_server.request_sync(session, input_pak, status, output)
assert status == cnis.Status.SUCCESS
assert custom_python_preproc.called
# Destroy session
assert infer_server.destroy_session(session)
def test_data_layout():
infer_server =cnis.InferServer(dev_id=0)
# Load model
model = infer_server.load_model(utils.model_dir)
# Check model input and output data type and dim order
if cnis.get_device_core_version(dev_id=0) == cnis.CoreVersion.MLU270 or \
cnis.get_device_core_version(dev_id=0) == cnis.CoreVersion.MLU220:
assert model.input_layout(0).dtype == cnis.DataType.UINT8
assert model.input_layout(0).order == cnis.DimOrder.NHWC
assert model.output_layout(0).dtype == cnis.DataType.FLOAT16
assert model.output_layout(0).order == cnis.DimOrder.NHWC
else:
assert model.input_layout(0).dtype == cnis.DataType.UINT8
assert model.input_layout(0).order == cnis.DimOrder.NHWC
assert model.output_layout(0).dtype == cnis.DataType.FLOAT32
assert model.output_layout(0).order == cnis.DimOrder.NONE
assert model.output_layout(1).dtype == cnis.DataType.INT32
assert model.output_layout(1).order == cnis.DimOrder.NONE
# Check data type size is correct
assert cnis.get_type_size(cnis.DataType.UINT8) == 1
assert cnis.get_type_size(cnis.DataType.FLOAT32) == 4
assert cnis.get_type_size(cnis.DataType.FLOAT16) == 2
assert cnis.get_type_size(cnis.DataType.INT16) == 2
assert cnis.get_type_size(cnis.DataType.INT32) == 4
def test_create_sync_session():
# First of all we need to create an InferServer object
infer_server = cnis.InferServer(dev_id=0)
# Secondly, to create a synchronous session, a session description is needed.
session_desc = cnis.SessionDesc()
session_desc.name = "test_session"
# Load model and set it to session description
session_desc.model = infer_server.load_model(utils.model_dir)
session_desc.show_perf = False
# create a synchronous session
session = infer_server.create_sync_session(session_desc)
assert session
# Test request sync
input_pak = prepare_input(session_desc.model)
output = cnis.Package(1, utils.tag)
status = cnis.Status.SUCCESS
assert infer_server.request_sync(session, input_pak, status, output)
assert status == cnis.Status.SUCCESS
assert output.perf
assert len(output.data) == 1 and output.data[0].get_model_io()
# Test get model and unload model
assert infer_server.get_model(session) == session_desc.model
assert cnis.InferServer.unload_model(session_desc.model)
assert not cnis.InferServer.unload_model(session_desc.model)
model = infer_server.load_model(utils.model_dir)
cnis.InferServer.clear_model_cache()
assert not cnis.InferServer.unload_model(model)
# destroy session
assert infer_server.destroy_session(session)
def test_observer():
infer_server = cnis.InferServer(dev_id=0)
session_desc = cnis.SessionDesc()
session_desc.model = infer_server.load_model(utils.model_dir)
session_desc.show_perf = False
class TestPostproc(cnis.Postprocess):
"""To use custom postprocess, we define a class TestPostproc which inherits from cnis.Postprocess.
The execute_func API will be called by InferServer to do postprocess.
"""
def __init__(self):
super().__init__()
def execute_func(self, result, model_output, model_info):
result.set({"key1": "result1", "key2": "result2"})
return True
session_desc.postproc = cnis.Postprocessor()
session_desc.set_postproc_func(TestPostproc().execute)
class TestObserver(cnis.Observer):
"""To receive results from InferServer, we define a class TestObserver which inherits from cnis.Observer.
After a request is sent to InferServer and is processed by InferServer, the response_func API will be called with
status, results and user data.
"""
def __init__(self):
super().__init__()
self.called = False
def response_func(self, status, data, user_data):
# Check user data
assert "user_data" in user_data
assert user_data["user_data"] == "cnis"
assert len(data.data) == 1
# Check data
result = data.data[0].get_dict()
assert "key1" in result
assert "key2" in result
assert result["key1"] == "result1"
assert result["key2"] == "result2"
self.called = True
obs = TestObserver()
session = infer_server.create_session(session_desc, obs)
input_pak = cnis.Package(1, utils.tag)
if cnis.get_device_core_version(dev_id=0) == cnis.CoreVersion.MLU270 or \
cnis.get_device_core_version(dev_id=0) == cnis.CoreVersion.MLU220:
input_pak.data[0].set(
np.random.randint(0, 255, size=(300, 300, 4), dtype=np.uint8))
else:
input_pak.data[0].set(
np.random.randint(0, 255, size=(416, 416, 3), dtype=np.uint8))
infer_server.request(session, input_pak, {"user_data": "cnis"})
infer_server.wait_task_done(session, utils.tag)
assert obs.called
infer_server.destroy_session(session)
def test_video_preprocessor_mlu():
# Create VideoPreprocessorMLU
preproc = cnis.VideoPreprocessorMLU()
assert preproc
# Create InferServer object
infer_server = cnis.InferServer(dev_id=0)
session_desc = cnis.SessionDesc()
session_desc.name = "test_session"
# Load model
session_desc.model = infer_server.load_model(utils.model_dir)
# Create VideoPreprocessorMLU and set parameters. Use CNCV preproc.
session_desc.preproc = preproc
if cnis.get_device_core_version(dev_id=0) == cnis.CoreVersion.MLU270 or \
cnis.get_device_core_version(dev_id=0) == cnis.CoreVersion.MLU220:
session_desc.set_preproc_params(
cnis.VideoPixelFmt.BGRA,
cnis.VideoPreprocessType.CNCV_PREPROC,
keep_aspect_ratio=False)
else:
session_desc.set_preproc_params(
cnis.VideoPixelFmt.RGB24,
cnis.VideoPreprocessType.CNCV_PREPROC,
keep_aspect_ratio=True)
session_desc.show_perf = False
# Create synchronous session
session = infer_server.create_sync_session(session_desc)
# Request
request_package(infer_server, session)
# Destroy sessoion
infer_server.destroy_session(session)
# Use RCOP preproc.
if cnis.get_device_core_version(dev_id=0) == cnis.CoreVersion.MLU270 or \
cnis.get_device_core_version(dev_id=0) == cnis.CoreVersion.MLU220:
session_desc.set_preproc_params(
cnis.VideoPixelFmt.BGRA,
cnis.VideoPreprocessType.RESIZE_CONVERT,
keep_aspect_ratio=False)
# Create synchronous session
session = infer_server.create_sync_session(session_desc)
# Request
request_package(infer_server, session)
# Destroy sessoion
infer_server.destroy_session(session)
def execute(self):
"""Execute asynchronous demo"""
# Create InferServer
cnis.bind_device(self.dev_id)
infer_server = cnis.InferServer(self.dev_id)
# Create session. Sync API
session_desc = cnis.SessionDesc()
session_desc.name = "test_session_async"
session_desc.engine_num = 1
session_desc.strategy = cnis.BatchStrategy.DYNAMIC
# Load model
session_desc.model = infer_server.load_model(self.model_dir)
# Set preproc and postproc
self.cncv_preproc(session_desc)
self.cpp_postproc(session_desc)
# Create observer
obs = self.MyObserver()
# Create session
session = infer_server.create_session(session_desc, obs)
for _ in range(4):
input_pak = self.prepare_mlu_data()
user_data = {"user_data": "this is user data"}
# Request
ret = infer_server.request(session,
input_pak,
user_data,
timeout=20000)
if not ret:
print(
"[EasyDK PyAPISamples] [AsyncDemo] RequestAsync failed, ret: {}"
.format(ret))
# Wait all tasks done
infer_server.wait_task_done(session, self.tag)
# Destroy Session
infer_server.destroy_session(session)
def test_create_session():
# First of all we need to create an InferServer object
infer_server = cnis.InferServer(dev_id=0)
# Secondly, to create an asynchronous session, a session description is needed.
session_desc = cnis.SessionDesc()
session_desc.name = "test_session"
# Load model and set it to session description
session_desc.model = infer_server.load_model(utils.model_dir)
session_desc.show_perf = False
# Define a TestObserver class for receiving results. Create a TestObserver object and set it to description.
class TestObserver(cnis.Observer):
"""To receive results from InferServer, we define a class TestObserver which inherits from cnis.Observer.
After a request is sent to InferServer and is processed by InferServer, the response_func API will be called with
status, results and user data.
"""
def __init__(self):
super().__init__()
self.called = False
def response_func(self, status, data, user_data):
assert status == cnis.Status.SUCCESS
assert "user_data" in user_data
assert user_data["user_data"] == "cnis"
self.called = True
obs = TestObserver()
# create an asynchronous session
session = infer_server.create_session(session_desc, obs)
assert session
# Test request
input_pak = prepare_input(session_desc.model)
assert infer_server.request(session, input_pak, {"user_data":"cnis"})
infer_server.wait_task_done(session, utils.tag)
assert obs.called
# Test discard task
for _ in range(10):
input_pak = prepare_input(session_desc.model)
assert infer_server.request(session, input_pak, {"user_data":"cnis"})
infer_server.discard_task(session, utils.tag)
infer_server.wait_task_done(session, utils.tag)
# destroy session
assert infer_server.destroy_session(session)
def test_session():
infer_server = cnis.InferServer(dev_id=0)
session_desc = cnis.SessionDesc()
session_desc.name = "test_session"
session_desc.model = infer_server.load_model(utils.model_dir)
session_desc.strategy = cnis.BatchStrategy.DYNAMIC
session_desc.preproc = cnis.PreprocessorHost()
session_desc.set_preproc_func(CustomPreprocess().execute)
session_desc.postproc = cnis.Postprocessor()
session_desc.set_postproc_func(CustomPostprocess().execute)
session_desc.host_input_layout = cnis.DataLayout(
cnis.DataType.UINT8, cnis.DimOrder.NHWC)
session_desc.host_output_layout = cnis.DataLayout(
cnis.DataType.FLOAT32, cnis.DimOrder.NHWC)
session_desc.batch_timeout = 1000
session_desc.priority = 0
session_desc.engine_num = 2
session_desc.show_perf = False
session = infer_server.create_sync_session(session_desc)
assert session
assert infer_server.destroy_session(session)
class TestObserver(cnis.Observer):
"""To receive results from InferServer, we define a class TestObserver which inherits from cnis.Observer.
After a request is sent to InferServer and is processed by InferServer, the response_func API will be called with
status, results and user data.
"""
def __init__(self):
super().__init__()
def response_func(self, status, data, user_data):
# Empty, just for testing session creation
pass
obs = TestObserver()
session = infer_server.create_session(session_desc, obs)
assert session
assert infer_server.destroy_session(session)
def execute(self):
"""Execyte synchronous demo"""
# Create InferServer
cnis.bind_device(self.dev_id)
infer_server = cnis.InferServer(self.dev_id)
# Create session. Sync API
session_desc = cnis.SessionDesc()
session_desc.name = "test_session_sync"
session_desc.engine_num = 1
session_desc.strategy = cnis.BatchStrategy.STATIC
session_desc.model = infer_server.load_model(self.model_dir)
self.opencv_preproc(session_desc)
self.python_postproc(session_desc)
session = infer_server.create_sync_session(session_desc)
for _ in range(4):
# Prepare input and output
input_pak = self.prepare_opencv_data()
output_pak = cnis.Package(1)
# Request
status = cnis.Status.SUCCESS
ret = infer_server.request_sync(session,
input_pak,
status,
output_pak,
timeout=20000)
if not ret:
print(
"[EasyDK PythonAPISamples] [SyncDemo] RequestSync failed, ret: {}, status: {}"
.format(ret, status))
if status == cnis.Status.SUCCESS:
self.print_result(output_pak)
# Destroy Session
infer_server.destroy_session(session)
def test_model():
infer_server = cnis.InferServer(dev_id=0)
model = infer_server.load_model(utils.model_dir)
if cnis.get_device_core_version(dev_id=0) == cnis.CoreVersion.MLU270 or \
cnis.get_device_core_version(dev_id=0) == cnis.CoreVersion.MLU220:
assert model.input_layout(0)
assert model.input_shape(0)
assert model.output_layout(0)
assert model.output_shape(0)
assert model.input_num() == 1
assert model.output_num() == 1
assert model.batch_size() == 4
assert model.get_key() == "./vgg16_ssd_b4c4_bgra_mlu270.cambricon_subnet0"
else:
assert model.input_layout(0)
assert model.input_shape(0)
assert model.output_layout(0)
assert model.output_shape(0)
assert model.output_layout(1)
assert model.output_shape(1)
assert model.input_num() == 1
assert model.output_num() == 2
assert model.batch_size() == 4
assert model.get_key() == "./yolov3_nhwc_tfu_0.8.2_uint8_int8_fp16.model"