def _output_dataset_to_numpy(self):
dataset = []
output_tensor_list = self._gen_output_tensor()
num = acl.mdl.get_dataset_num_buffers(self._output_dataset)
for i in range(num):
buf = acl.mdl.get_dataset_buffer(self._output_dataset, i)
data = acl.get_data_buffer_addr(buf)
size = int(acl.get_data_buffer_size(buf))
output_ptr = output_tensor_list[i]["ptr"]
output_data = output_tensor_list[i]["tensor"]
if isinstance(output_data, bytes):
data_size = len(output_data)
else:
data_size = output_data.size * output_data.itemsize
ret = acl.rt.memcpy(output_ptr, data_size, data, size,
self._copy_policy)
if ret != const.ACL_SUCCESS:
log_error("Memcpy inference output to local failed")
return None
if isinstance(output_data, bytes):
output_data = np.frombuffer(
output_data, dtype=output_tensor_list[i]["dtype"]).reshape(
output_tensor_list[i]["shape"])
output_tensor = output_data.copy()
else:
output_tensor = output_data
dataset.append(output_tensor)
return dataset
def _parse_input_data(self, input_data, index):
data = None
size = 0
if isinstance(input_data, AclLiteImage):
size = input_data.size
data = input_data.data()
elif isinstance(input_data, np.ndarray):
size = input_data.size * input_data.itemsize
if "bytes_to_ptr" in dir(acl.util):
bytes_data = input_data.tobytes()
ptr = acl.util.bytes_to_ptr(bytes_data)
else:
ptr = acl.util.numpy_to_ptr(input_data)
data = self._copy_input_to_device(ptr, size, index)
if data is None:
size = 0
log_error("Copy input to device failed")
elif (isinstance(input_data, dict) and ('data' in input_data.keys())
and ('size' in input_data.keys())):
size = input_data['size']
data = input_data['data']
else:
log_error("Unsupport input")
return data, size
def execute(self, input_list):
"""
inference input data
Args:
input_list: input data list, support AclLiteImage,
numpy array and {'data': ,'size':} dict
returns:
inference result data, which is a numpy array list,
each corresponse to a model output
"""
ret = self._gen_input_dataset(input_list)
if ret == const.FAILED:
log_error("Gen model input dataset failed")
return None
ret = acl.mdl.execute(self._model_id, self._input_dataset,
self._output_dataset)
if ret != const.ACL_SUCCESS:
log_error("Execute model failed for acl.mdl.execute error ", ret)
return None
self._release_dataset(self._input_dataset)
self._input_dataset = None
return self._output_dataset_to_numpy()
def recv_msg(self):
"""Receive message from presenter server
Returns:
msg_name: received message name
msg_body: received message data
"""
# Step1: read msg head
msg_total_len, msg_name_len = self._read_msg_head(5)
if msg_total_len is None:
log_error("msg total len is None.")
return None, None
# Step2: read msg name
ret, msg_name = self._read_msg_name(msg_name_len)
if not ret:
return None, None
# Step3: read msg body
msg_body_len = msg_total_len - 5 - msg_name_len
if msg_body_len < 0:
log_error("msg total len is 0")
return None, None
ret, msg_body = self._read_msg_body(msg_body_len)
if not ret:
return None, None
return msg_name, msg_body
def __init__(self, image, width=0, height=0, alignWidth=0, alignHeight=0,
size=0, memory_type=const.MEMORY_NORMAL):
"""Create AclLiteImage instance
Args:
image: image data, binary, numpy array or file path
width: image width. if image is jpeg or png file,
this arg is not nesscessary
height: image height. if image is jpeg or png file, this arg is
not nesscessary
size: image data size. if image is file path, this arg is not
nesscessary
memory_type: memory type of image data. if image is file path, this
arg is not nesscessary
"""
self._data = None
self._memory_type = memory_type
self.width = 0
self.height = 0
self.alignWidth = 0
self.alignHeight = 0
self.size = 0
self._encode_format = const.ENCODE_FORMAT_UNKNOW
self._load_ok = True
if isinstance(image, str):
self._instance_by_image_file(image, width, height)
elif isinstance(image, int):
self._instance_by_buffer(image, width, height, size)
elif isinstance(image, np.ndarray):
self._instance_by_nparray(image, width, height)
else:
acl_log.log_error("Create instance failed for "
"unknow image data type")
def read(self, no_wait=False):
"""Read decoded frame
no_wait: Get image without wait. If set this arg True, and
return image is None, should call is_finished() method
to confirm decode finish or failed
Returns:
1. const.SUCCESS, not None: get image success
2. const.SUCCESS, None: all frames decoded and be token off
3. const.FAILED, None: Has frame not decoded, but no image decoded,
it means decode video failed
"""
image = None
ret = const.SUCCESS
# received eos frame and all received frame decode complete
if no_wait or self.is_finished():
try:
image = self._frame_queue.get_nowait()
except queue.Empty:
acl_log.log_info("No decode frame in queue anymore")
else:
try:
image = self._frame_queue.get(timeout=READ_TIMEOUT)
except queue.Empty:
ret = const.FAILED
acl_log.log_error("Read channel id %d frame timeout, "
"receive frame %d, decoded %d"
% (self._channel_id, self._decode_cnt,
self._decode_complete_cnt))
return ret, image
def _pyav_vdec(self):
frame = 0
video = av.open(self._stream_name)
stream = [s for s in video.streams if s.type == 'video']
acl_log.log_info("Start decode %s frames" % (self._stream_name))
for packet in video.demux([stream[0]]):
# Get frame data from packet and copy to dvpp
frame_data, data_size = self._prepare_frame_data(packet)
if data_size == 0:
# Last packet size is 0, no frame to decode anymore
break
if self._vdec.process(frame_data, data_size,
[self._channel_id, frame]):
acl_log.log_error("Dvpp vdec deocde frame %d failed, "
"stop decode" % (frame))
self._status = DECODE_STATUS_ERROR
break
frame += 1
# The status chang to stop when app stop decode
if self._status != DECODE_STATUS_RUNNING:
acl_log.log_info("Decode status change to %d, stop decode" %
(self._status))
break
def __init__(self, strame_name):
self._stream_name = strame_name
self._input_buffer = None
self._vdec = None
self._is_opened = False
self._width = 0
self._height = 0
self._decode_thread_id = None
self._dextory_dvpp_flag = False
self._ctx, ret = acl.rt.get_context()
if ret:
acl_log.log_error(
"Get acl context failed when "
"instance AclVideo, error ", ret)
else:
self._entype = const.ENTYPE_H264_MAIN
self._channel_id = gen_unique_channel_id()
self._status = DECODE_STATUS_INIT
self._run_mode, ret = acl.rt.get_run_mode()
if ret:
acl_log.log_error(
"Get acl run mode failed when "
"instance AclVideo, error ", ret)
else:
self._open()
def _get_param(self):
container = av.open(self._stream_name)
stream = [s for s in container.streams if s.type == 'video']
if len(stream) == 0:
# The stream is not video
acl_log.log_error("%s has no video stream" % (self._stream_name))
return const.FAILED
ret, profile = self._get_profile(stream)
if ret:
acl_log.log_error("%s is not annex-b format, decode failed" %
(self._stream_name))
return const.FAILED
video_context = container.streams.video[0].codec_context
codec_id_name = video_context.name
ret, self._entype = self._get_entype(codec_id_name, profile)
if ret:
return const.FAILED
self._width = video_context.width
self._height = video_context.height
acl_log.log_info("Get %s infomation: width %d, height %d, profile %d, "
"codec %s, entype %d" %
(self._stream_name, self._width, self._height,
profile, codec_id_name, self._entype))
container.close()
return const.SUCCESS
def _prepare_frame_data(self, packet):
in_frame_np = np.frombuffer(packet.to_bytes(), np.byte)
size = in_frame_np.size
if size == 0:
# Last frame data is empty
acl_log.log_info("Pyav decode finish")
self._status = DECODE_STATUS_PYAV_FINISH
return None, 0
if "bytes_to_ptr" in dir(acl.util):
bytes_data = in_frame_np.tobytes()
in_frame_ptr = acl.util.bytes_to_ptr(bytes_data)
else:
in_frame_ptr = acl.util.numpy_to_ptr(in_frame_np)
policy = const.ACL_MEMCPY_DEVICE_TO_DEVICE
if self._run_mode == const.ACL_HOST:
policy = const.ACL_MEMCPY_HOST_TO_DEVICE
ret = acl.rt.memcpy(self._input_buffer, size, in_frame_ptr, size,
policy)
if ret:
acl_log.log_error("Copy data to dvpp failed, policy %d, error %d" %
(policy, ret))
self._status = DECODE_STATUS_ERROR
return None, 0
return self._input_buffer, size
def jpegd(self, image):
"""
jepg image to yuv image
"""
stride_width = utils.align_up128(image.width)
stride_height = utils.align_up16(image.height)
stride_size = utils.yuv420sp_size(stride_width, stride_height)
# Create conversion output image desc
output_desc, out_buffer = self._gen_jpegd_out_pic_desc(
image, stride_size)
ret = acl.media.dvpp_jpeg_decode_async(self._dvpp_channel_desc,
image.data(), image.size,
output_desc, self._stream)
if ret != constants.ACL_SUCCESS:
log_error("dvpp_jpeg_decode_async failed ret={}".format(ret))
return None
ret = acl.rt.synchronize_stream(self._stream)
if ret != constants.ACL_SUCCESS:
log_error("dvpp_jpeg_decode_async failed ret={}".format(ret))
return None
# Return the decoded AclLiteImage instance
return AclLiteImage(out_buffer, image.width, image.height,
stride_width, stride_height, stride_size,
constants.MEMORY_DVPP)
def main():
"""
Function description:
Main function
"""
acl_resource = AclLiteResource()
acl_resource.init()
detector = Yolov3(acl_resource, MODEL_PATH, MODEL_WIDTH, MODEL_HEIGHT)
video_decoders, postprocessor = create_threads(detector)
if video_decoders is None:
log_error("Please check the configuration in %s is valid" %
(COCO_DETEC_CONF))
return
while True:
all_process_fin = True
for decoder in video_decoders:
ret, data = decoder.get_data()
if ret == False:
continue
if data:
detect_results = detector.execute(data)
postprocessor.process(data, detect_results)
all_process_fin = False
if all_process_fin:
log_info("all video decoder finish")
break
postprocessor.exit()
log_info("sample execute end")
def _open(self):
ret = libacllite.OpenCameraEx(self._id, self._fps, self._width,
self._height)
if (ret != CAMERA_OK):
log_error("Open camera %d failed ,ret = %d" % (self._id, ret))
return CAMERA_ERROR
self._status = CAMERA_OPENED
return CAMERA_OK
def copy_to_dvpp(self):
"""Copy image data to dvpp"""
device_ptr = utils.copy_data_to_dvpp(self.data(), self.size,
self._run_mode)
if device_ptr is None:
acl_log.log_error("Copy image to dvpp failed")
return None
return AclLiteImage(device_ptr, self.width, self.height, 0, 0,
self.size, const.MEMORY_DVPP)
def _gen_resize_out_pic_desc(self, resize_width, resize_height,
output_size):
out_buffer, ret = acl.media.dvpp_malloc(output_size)
if ret != constants.ACL_SUCCESS:
log_error("AclLiteImageProc malloc failed, error: ", ret)
return None
pic_desc = self._gen_output_pic_desc(resize_width, resize_height,
out_buffer, output_size)
return pic_desc, out_buffer
def _release_databuffer(self, data_buffer, free_memory=False):
if free_memory:
data_addr = acl.get_data_buffer_addr(data_buffer)
if data_addr:
acl.rt.free(data_addr)
ret = acl.destroy_data_buffer(data_buffer)
if ret != const.ACL_SUCCESS:
log_error("Destroy data buffer error ", ret)
def read(self):
"""Read frame from camera"""
frame_data = CameraOutputC()
ret = libacllite.ReadCameraFrame(self._id, byref(frame_data))
if (ret != CAMERA_OK):
log_error("Read camera %d failed" % (self._id))
return None
return AclLiteImage(addressof(frame_data.data.contents), self._width,
self._height, 0, 0, self._size, const.MEMORY_DVPP)
def _keep_alive(self):
msg = pm.heartbeat_message()
while True:
if self._closed:
log_error("Heard beat thread exit")
break
self.socket.send_msg(msg)
time.sleep(2)
def open_channel(config_file):
"""Connect with presenter server"""
server_ip, port, channel_name, content_type = get_channel_config(
config_file)
channel = PresenterChannel(server_ip, port, channel_name, content_type)
ret = channel.startup()
if ret:
log_error("ERROR:Open channel failed")
return None
return channel
def _create_input_pic_stream_desc(self, input_data, input_size):
stream_desc = acl.media.dvpp_create_stream_desc()
if stream_desc is None:
acl_log.log_error("Create dvpp vdec input pic stream desc failed")
return None
acl.media.dvpp_set_stream_desc_size(stream_desc, input_size)
acl.media.dvpp_set_stream_desc_data(stream_desc, input_data)
return stream_desc
def _get_jpegd_memory_size(self, image, stride_size):
if image.is_local():
size, ret = acl.media.dvpp_jpeg_predict_dec_size(
image.data(), image.size,
constants.PIXEL_FORMAT_YUV_SEMIPLANAR_420)
if ret != constants.ACL_SUCCESS:
log_error("Predict jpeg decode size failed, return ", ret)
return False, 0
return True, size
else:
return True, int(stride_size)
def _read_msg_head(self, read_len):
ret, msg_head = self._read_socket(read_len)
if not ret:
log_error("socket receive msg head null")
return None, None
# in Struct(), 'I' is unsigned int, 'B' is unsigned char
msg_head_data = struct.Struct('IB')
(msg_total_len, msg_name_len) = msg_head_data.unpack(msg_head)
msg_total_len = socket.ntohl(msg_total_len)
return msg_total_len, msg_name_len
def _get_image_format_by_suffix(self, filename):
suffix = os.path.splitext(filename)[-1].strip().lower()
if (suffix == ".jpg") or (suffix == ".jpeg"):
image_format = const.ENCODE_FORMAT_JPEG
elif suffix == ".png":
image_format = const.ENCODE_FORMAT_PNG
elif suffix == ".yuv":
image_format = const.ENCODE_FORMAT_YUV420_SP
else:
acl_log.log_error("Unsupport image format: ", suffix)
image_format = const.ENCODE_FORMAT_UNKNOW
return image_format
def _read_msg_name(self, msg_name_len):
ret, msg_name = self._read_socket(msg_name_len)
if not ret:
log_error("Socket receive msg but name is null")
return False, None
try:
msg_name = msg_name.decode("utf-8")
except:
log_error("Msg name decode to utf-8 error")
return False, None
return True, msg_name
def _release_dataset(self, dataset, free_memory=False):
if not dataset:
return
num = acl.mdl.get_dataset_num_buffers(dataset)
for i in range(num):
data_buf = acl.mdl.get_dataset_buffer(dataset, i)
if data_buf:
self._release_databuffer(data_buf, free_memory)
ret = acl.mdl.destroy_dataset(dataset)
if ret != const.ACL_SUCCESS:
log_error("Destroy data buffer error ", ret)
def _get_profile(self, stream):
# Annex-b format h264 extradata is start with 0x000001 or 0x00000001
extradata = np.frombuffer(stream[0].codec_context.extradata, np.ubyte)
if (extradata[0:3] == [0, 0, 1]).all():
profile_id = extradata[4]
elif (extradata[0:4] == [0, 0, 0, 1]).all():
profile_id = extradata[5]
else:
acl_log.log_error("The stream %s is not annex-b h264, "
"can not decode it" % (self._stream_name))
return const.FAILED, None
return const.SUCCESS, profile_id
def _get_pic_desc_data(self, pic_desc, user_data):
pic_data = acl.media.dvpp_get_pic_desc_data(pic_desc)
pic_data_size = acl.media.dvpp_get_pic_desc_size(pic_desc)
ret_code = acl.media.dvpp_get_pic_desc_ret_code(pic_desc)
if ret_code:
channel_id, frame_id = user_data
acl_log.log_error("Decode channel %d frame %d failed, error %d"
% (channel_id, frame_id, ret_code))
acl.media.dvpp_free(pic_data)
else:
image = AclLiteImage(pic_data, self._width, self._height, 0, 0,
pic_data_size, const.MEMORY_DVPP)
self._frame_queue.put(image)
acl.media.dvpp_destroy_pic_desc(pic_desc)
def send_msg(self, data):
"""Send message to presenter server
Args:
data: message data
Returns:
0 send success
1 send failed
"""
try:
self._sock_client.sendall(data)
except:
log_error("Send msg failed")
return 1
return 0
def _decode_thread_entry(self, arg_list):
# Set acl context for decode thread
if self._decode_thread_init():
acl_log.log_error("Decode thread init failed")
return const.FAILED
self._status = DECODE_STATUS_READY
while (self._status == DECODE_STATUS_READY):
time.sleep(WAIT_INTERVAL)
self._pyav_vdec()
self._decode_thread_join()
return const.SUCCESS
def destroy(self):
"""Release image memory"""
if (self._data is None) or (self.size == 0):
acl_log.log_error("Release image abnormaly, data is None")
return
if self._memory_type == const.MEMORY_DEVICE:
acl.rt.free(self._data)
elif self._memory_type == const.MEMORY_HOST:
acl.rt.free_host(self._data)
elif self._memory_type == const.MEMORY_DVPP:
acl.media.dvpp_free(self._data)
# numpy no need release
self._data = None
self.size = 0