def run():
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("mask")
# 初始化摄像头
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
# 初始化模型
mask_model_path = hilens.get_model_dir() + "convert-mask-detection.om"
mask_model = hilens.Model(mask_model_path)
while True:
##### 1. 设备接入 #####
input_yuv = camera.read() # 读取一帧图片(YUV NV21格式)
##### 2. 数据预处理 #####
img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2RGB_NV21) # 转为RGB格式
img_preprocess, img_w, img_h = preprocess(img_rgb) # 缩放为模型输入尺寸
##### 3. 模型推理 #####
output = mask_model.infer([img_preprocess.flatten()])
##### 4. 结果输出 #####
bboxes = get_result(output, img_w, img_h) # 获取检测结果
img_rgb = draw_boxes(img_rgb, bboxes) # 在图像上画框
output_yuv = hilens.cvt_color(img_rgb, hilens.RGB2YUV_NV21)
display.show(output_yuv) # 显示到屏幕上
hilens.terminate()
def run_inner(work_path):
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("heart")
# 初始化摄像头与显示器
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
# 初始化模型
model_path = os.path.join(work_path, 'model/convert-6ecb.om')
cls_model = hilens.Model(model_path)
while True:
try:
# 2.1 读取摄像头数据
input_nv21 = camera.read()
# 2.2 截取出一个正方形区域作为手势识别输入并做预处理
input_rgb, gesture_area = preprocess(input_nv21)
input_resized = cv2.resize(gesture_area, (net_size, net_size))
# 2.3 模型推理
outputs = cls_model.infer([input_resized.flatten()])
# 2.4 结果展示
process_predict_result(outputs, input_rgb)
output_nv21 = hilens.cvt_color(input_rgb, hilens.RGB2YUV_NV21)
display.show(output_nv21)
except Exception as e:
print(e)
break
def run():
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("pose")
# 初始化模型
pose_model_path = hilens.get_model_dir() + "pose_template_model.om"
pose_model = hilens.Model(pose_model_path)
# 初始化USB摄像头与HDMI显示器
camera = hilens.VideoCapture()
display_hdmi = hilens.Display(hilens.HDMI)
while True:
# 读取一帧图片(BGR格式)
input_yuv = camera.read()
# 图片预处理：转为BGR格式、裁剪/缩放为模型输入尺寸
input_bgr = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2BGR_NV21)
img_preprocess = preprocess(input_bgr)
# 模型推理
model_outputs = pose_model.infer([img_preprocess.flatten()])
# 从推理结果中解码出人体关键点并画在图像中
points = get_points(input_bgr, model_outputs)
img_data = draw_limbs(input_bgr, points)
# 输出处理后的图像到HDMI显示器，必须先转换成YUV NV21格式
output_nv21 = hilens.cvt_color(img_data, hilens.BGR2YUV_NV21)
display_hdmi.show(output_nv21)
hilens.terminate()
def run(work_path):
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("driving")
# 初始化自带摄像头与HDMI显示器,
# hilens studio中VideoCapture如果不填写参数，则默认读取test/camera0.mp4文件，
# 在hilens kit中不填写参数则读取本地摄像头
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
# 初始化模型
model_path = os.path.join(work_path, 'model/yolo3.om')
driving_model = hilens.Model(model_path)
frame_index = 0
json_bbox_list = []
json_data = {'info': 'det_result'}
while True:
frame_index += 1
try:
time_start = time.time()
# 1. 设备接入 #####
input_yuv = camera.read() # 读取一帧图片(YUV NV21格式)
# 2. 数据预处理 #####
img_bgr = cv2.cvtColor(input_yuv,
cv2.COLOR_YUV2BGR_NV21) # 转为BGR格式
img_preprocess, img_w, img_h = preprocess(img_bgr) # 缩放为模型输入尺寸
# 3. 模型推理 #####
output = driving_model.infer([img_preprocess.flatten()])
# 4. 获取检测结果 #####
bboxes = get_result(output, img_w, img_h)
# 5-1. [比赛提交作品用] 将结果输出到json文件中 #####
if len(bboxes) > 0:
json_bbox = convert_to_json(bboxes, frame_index)
json_bbox_list.append(json_bbox)
# 5-2. [调试用] 将结果输出到模拟器中 #####
img_bgr = draw_boxes(img_bgr, bboxes) # 在图像上画框
output_yuv = hilens.cvt_color(img_bgr, hilens.BGR2YUV_NV21)
display.show(output_yuv) # 显示到屏幕上
time_frame = 1000 * (time.time() - time_start)
hilens.info('----- time_frame = %.2fms -----' % time_frame)
except RuntimeError:
print('last frame')
break
# 保存检测结果
hilens.info('write json result to file')
result_filename = './result.json'
json_data['result'] = json_bbox_list
save_json_to_file(json_data, result_filename)
hilens.terminate()
def __init__(self,
video_path="",
hilens_enable=True,
socket_enable=True,
display_enable=True,
model_enable=True,
record_enable=False):
self.__hilens_enable = hilens_enable
self.__socket_enable = socket_enable
self.__display_enable = display_enable
self.__model_enable = model_enable
self.__record_enable = record_enable
if self.__hilens_enable:
import hilens
video_path = ""
if self.__socket_enable:
self.__socket_3399 = SocketThreadGuard(socket_enable=socket_enable)
if self.__display_enable:
self.__display = hilens.Display(hilens.HDMI)
else:
self.__display = None
model = None
if self.__model_enable:
# 初始化模型
# model_path = hilens.get_model_dir() + "model-extreme-1-1.om"
if self.__hilens_enable:
model_path = hilens.get_model_dir() + "speed_detect.om"
else:
model_path = "model/convert-1360.om"
# model_path = hilens.get_model_dir() + "convert-1360.om"
model = hilens.Model(model_path)
print("create model")
self.__cam_thread = CameraThreadGuard(video_path,
RECORD=self.__record_enable,
HILENS=self.__hilens_enable)
self.__pp_thread = PictureProcessorThreadGuard(
model,
HILENS=self.__hilens_enable,
socket_enable=self.__socket_enable,
display=self.__display)
def __init__(self, path, RECORD=False, HILENS=False):
self.HILENS = HILENS
self.RECORD = RECORD
if HILENS: import hilens
self.disp = ""
if path != "":
self.cap = cv2.VideoCapture(path)
self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, 520)
self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 360)
else:
self.cap = hilens.VideoCapture()
self.disp = hilens.Display(hilens.HDMI)
if RECORD:
fps = 20
size = (520, 360)
format = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')
self.src_video_writer = cv2.VideoWriter("src_output.avi", format,
fps, size)
def main():
""" 利用SkillFramework进行人脸检测模型的推理 """
hilens.init("test") # 参数要与创建技能时填写的检验值保持一致
model_path = hilens.get_model_dir() + "face_detection_demo.om" # 模型路径
model = hilens.Model(model_path)
display_hdmi = hilens.Display(hilens.HDMI) # 图像通过hdmi输出到屏幕
camera = hilens.VideoCapture()
while True:
# 1. 读取摄像头输入（yuv nv21）
input_nv21 = camera.read()
# 2. 转为bgr
input_bgr = cv2.cvtColor(input_nv21, cv2.COLOR_YUV2BGR_NV21)
src_image_height = input_bgr.shape[0]
src_image_width = input_bgr.shape[1]
# 3. 保留原图比例的resize为网络输入尺寸
im_scale1 = float(input_width) / float(src_image_width)
im_scale2 = float(input_height) / float(src_image_height)
im_scale = min(im_scale1, im_scale2)
input_bgr_rescaled = cv2.resize(input_bgr, None, None, fx=im_scale, fy=im_scale)
input_bgr_resized = np.zeros((input_height, input_width, 3), dtype = np.uint8)
input_bgr_resized[0:input_bgr_rescaled.shape[0],0:input_bgr_rescaled.shape[1],:] = input_bgr_rescaled
# 3. 推理
outputs = model.infer([input_bgr_resized.flatten()])
# 4. 后处理得到人脸bounding box，恢复到原图比例，画人脸框
detect_boxes = im_detect_nms(outputs[0])
if len(detect_boxes) > 0:
for rect in detect_boxes:
left = max(rect[0] / im_scale, 0)
top = max(rect[1] / im_scale, 0)
right = min(rect[2] / im_scale, src_image_width)
bottom = min(rect[3] / im_scale, src_image_height)
cv2.rectangle(input_bgr, (int(left), int(top)), (int(right), int(bottom)), 255, 2)
# 5. 输出图像，必须是yuv nv21形式
output_nv21 = hilens.cvt_color(input_bgr, hilens.BGR2YUV_NV21)
display_hdmi.show(output_nv21)
def handler(a, b):
hilens.init("hello")
model = hilens.Model(model_path)
display_hdmi = hilens.Display(hilens.HDMI)
camera = hilens.VideoCapture()
while True:
input_nv21 = camera.read()
input_bgr = cv2.cvtColor(input_nv21,cv2.COLOR_YUV2BGR_NV21)
input_resized = cv2.resize(input_bgr, (net_w, net_h))
img_preprocess, img_w, img_h, new_w, new_h, shift_x_ratio, shift_y_ratio = preprocess(input_bgr, aipp_flag)
outputs = model.infer([input_resized.flatten()])
res = get_result(outputs, img_w, img_h, new_w, new_h, shift_x_ratio, shift_y_ratio)
img_data = draw_box_on_img(input_bgr, res)
output_nv21 = hilens.cvt_color(img_data, hilens.BGR2YUV_NV21)
display_hdmi.show(output_nv21)
def init():
global model, camera1, camera2, upload_uri1, upload_uri2, display_hdmi
hilens.init("hello")
model_path = hilens.get_model_dir() + "./convert-4label.om"
model = hilens.Model(model_path)
display_hdmi = hilens.Display(hilens.HDMI)
hilens.set_log_level(hilens.DEBUG)
skill_cfg = hilens.get_skill_config()
if skill_cfg is None or 'IPC_address1' not in skill_cfg or 'upload_uri1' not in skill_cfg:
hilens.fatal(
'Missing IPC1 skill configs! skill_cfg: {}'.format(skill_cfg))
hilens.terminate()
exit(1)
try:
camera1 = hilens.VideoCapture(skill_cfg['IPC_address1'])
except Exception as e:
hilens.fatal("Failed to create camera1 with {}, e: {}",
skill_cfg['IPC_address1'], e)
upload_uri1 = skill_cfg['upload_uri1']
if 'IPC_address2' not in skill_cfg or 'upload_uri2' not in skill_cfg:
hilens.warning(
'Missing IPC2 skill configs! Camera2 will not work. skill_cfg: {}'.
format(skill_cfg))
else:
try:
camera2 = hilens.VideoCapture(skill_cfg['IPC_address2'])
except Exception as e:
hilens.fatal("Failed to create camera2 with {}, e: {}",
skill_cfg['IPC_address2'], e)
upload_uri2 = skill_cfg['upload_uri2']
initModel(model)
def __init__(self, HILENS=True, RECORD=False, TrackBar=False):
self.__fps = 0
self.__fps_start = time.time()
if RECORD:
fps = 20
size = (520, 360)
format_ = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')
self.dst_video_writer = cv2.VideoWriter("dst_output.avi", format_,
fps, size)
if HILENS:
import hilens
self.disp = hilens.Display(hilens.HDMI)
self.threshold = 0.83
self.low_red_hsv = (170, 120, 120)
self.high_red_hsv = (180, 255, 255)
self.low_yellow_hsv = (15, 110, 110)
self.high_yellow_hsv = (30, 255, 255)
self.low_green_hsv = (80, 130, 130)
self.high_green_hsv = (90, 255, 255)
w = 520
h = 360
self.roi_rect = [(int(w * 1 / 4), int(h * 2 / 5)),
(int(w * 3 / 4), int(h * 2 / 3))]
if TrackBar:
cv2.namedWindow("param", cv2.WINDOW_NORMAL)
cv2.createTrackbar("l_h_r", "param", self.low_red_hsv[0], 180,
nothing)
cv2.createTrackbar("l_s_r", "param", self.low_red_hsv[1], 255,
nothing)
cv2.createTrackbar("l_v_r", "param", self.low_red_hsv[2], 255,
nothing)
cv2.createTrackbar("h_h_r", "param", self.high_red_hsv[0], 180,
nothing)
cv2.createTrackbar("h_s_r", "param", self.high_red_hsv[1], 255,
nothing)
cv2.createTrackbar("h_v_r", "param", self.high_red_hsv[2], 255,
nothing)
cv2.createTrackbar("l_h_y", "param", self.low_yellow_hsv[0], 180,
nothing)
cv2.createTrackbar("l_s_y", "param", self.low_yellow_hsv[1], 255,
nothing)
cv2.createTrackbar("l_v_y", "param", self.low_yellow_hsv[2], 255,
nothing)
cv2.createTrackbar("h_h_y", "param", self.high_yellow_hsv[0], 180,
nothing)
cv2.createTrackbar("h_s_y", "param", self.high_yellow_hsv[1], 255,
nothing)
cv2.createTrackbar("h_v_y", "param", self.high_yellow_hsv[2], 255,
nothing)
cv2.createTrackbar("l_h_g", "param", self.low_green_hsv[0], 180,
nothing)
cv2.createTrackbar("l_s_g", "param", self.low_green_hsv[1], 255,
nothing)
cv2.createTrackbar("l_v_g", "param", self.low_green_hsv[2], 255,
nothing)
cv2.createTrackbar("h_h_g", "param", self.high_green_hsv[0], 255,
nothing)
cv2.createTrackbar("h_s_g", "param", self.high_green_hsv[1], 255,
nothing)
cv2.createTrackbar("h_v_g", "param", self.high_green_hsv[2], 255,
nothing)
self.TrackBar = TrackBar
self.HILENS = HILENS
self.RECORD = RECORD
template_paths = [
"Images/red_template_1.jpg", "Images/yellow_template_1.jpg"
]
self.templates = list()
for path in template_paths:
template = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
# self.templates[-1] = cv2.convertScaleAbs(self.templates[-1], alpha=1.5, beta=-100)
self.templates.append(cv2.resize(template, (9, 9)))
self.templates.append(cv2.resize(template, (15, 15)))
self.templates.append(cv2.resize(template, (21, 21)))
self.templates.append(cv2.resize(template, (27, 27)))
self.templates.append(cv2.resize(template, (33, 33)))
for t in self.templates:
print(t.shape)
self.labels = ["red_stop", "green_go", "yellow_back"]
self.colors = [(0, 0, 255), (0, 255, 0), (0, 255, 255)]
def run():
# 配置系统日志级别
hilens.set_log_level(hilens.ERROR)
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("gesture")
# 初始化模型
gesture_model_path = hilens.get_model_dir() + "gesture_template_model.om"
gesture_model = hilens.Model(gesture_model_path)
# 初始化本地摄像头与HDMI显示器
camera = hilens.VideoCapture()
display_hdmi = hilens.Display(hilens.HDMI)
# 上一次上传OBS图片的时间与上传间隔
last_upload_time = 0
upload_duration = 5
# 读取技能配置
skill_cfg = hilens.get_skill_config()
if skill_cfg is None or 'server_url' not in skill_cfg:
hilens.error("server_url not configured")
return
while True:
# 读取一帧图片(YUV NV21格式)
input_yuv = camera.read()
# 图片预处理：转为RGB格式、缩放为模型输入尺寸
img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2RGB_NV21)
img_preprocess, img_w, img_h = preprocess(img_rgb)
# 模型推理
output = gesture_model.infer([img_preprocess.flatten()])
# 后处理得到手势所在区域与类别，并在RGB图中画框
bboxes = get_result(output, img_w, img_h)
img_rgb = draw_boxes(img_rgb, bboxes)
# 输出处理后的图像到HDMI显示器，必须先转回YUV NV21格式
output_yuv = hilens.cvt_color(img_rgb, hilens.RGB2YUV_NV21)
display_hdmi.show(output_yuv)
# 上传OK手势图片到OBS，为防止OBS数据存储过多，间隔一定的时间才上传图片
if time.time() - last_upload_time > upload_duration:
# 截取出OK手势图片(如果有的话)
img_OK = get_OK(img_rgb, bboxes)
if img_OK is not None:
# 上传OK手势图片到OBS，图片(用当前时间命名)需要先转为BGR格式并按照jpg格式编码
img_OK = cv2.cvtColor(img_OK, cv2.COLOR_RGB2BGR)
img_OK = cv2.imencode('.jpg', img_OK)[1]
filename = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime())
ret = hilens.upload_bufer(filename + "_OK.jpg", img_OK,
"write")
if ret != 0:
hilens.error("upload pic failed!")
return
last_upload_time = time.time()
# 以POST方式传输处理后的整张图片
try:
post_msg(skill_cfg['server_url'], img_rgb)
except Exception as e:
hilens.error("post data failed!")
print("Reason : ", e)
hilens.terminate()
def run():
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("landmarks")
# 初始化自带摄像头与HDMI显示器
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
# 初始化模型：人脸检测模型(centerface)、人脸68个关键点检测模型
centerface_model_path = hilens.get_model_dir() + "centerface_template_model.om"
centerface_model = hilens.Model(centerface_model_path)
landmark_model_path = hilens.get_model_dir() + "landmark68_template_model.om"
landmark_model = hilens.Model(landmark_model_path)
# 本段代码展示如何录制HiLens Kit摄像头拍摄的视频
fps = 10
size = (1280, 720)
format = cv2.VideoWriter_fourcc('M','J','P','G') # 注意视频格式
writer = cv2.VideoWriter("face.avi", format, fps, size)
# 待保存视频的起始帧数，可自行调节或加入更多逻辑
frame_count = 0
frame_start = 100
frame_end = 150
uploaded = False
while True:
# 读取一帧图片(YUV NV21格式)
input_yuv = camera.read()
# 图片预处理：转为RGB格式、缩放为模型输入尺寸
img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2RGB_NV21)
img_pre = preprocess(img_rgb)
img_h, img_w = img_rgb.shape[:2]
# 人脸检测模型推理，并进行后处理得到画面中最大的人脸检测框
output = centerface_model.infer([img_pre.flatten()])
face_box = get_largest_face_box(output, img_h, img_w)
# 画面中检测到有人脸且满足一定条件
if face_box is not None:
# 截取出人脸区域并做预处理
img_face = preprocess_landmark(img_rgb, face_box)
# 人脸关键点模型推理，得到68个人脸关键点
output2 = landmark_model.infer([img_face.flatten()])
landmarks = output2[0].reshape(68, 2)
# 将人脸框和人脸关键点画在RGB图中
img_rgb = draw_landmarks(img_rgb, face_box, landmarks)
# 输出处理后的图像到HDMI显示器，必须先转换成YUV NV21格式
output_nv21 = hilens.cvt_color(img_rgb, hilens.RGB2YUV_NV21)
display.show(output_nv21)
# 录制一段视频并发送到OBS中
if not uploaded:
frame_count += 1
if frame_count > frame_end: # 录制结束点
uploaded = True
writer.release() # 先保存在本地
ret = hilens.upload_file("face.avi", "face.avi", "write") # 发送到OBS中
if ret != 0:
hilens.error("upload file failed!")
return
elif frame_count > frame_start: # 录制开始点
# 注意写入的图片格式必须为BGR
writer.write(cv2.cvtColor(img_rgb, cv2.COLOR_RGB2BGR))
hilens.terminate()
def run(work_path):
global data
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("driving")
# 初始化自带摄像头与HDMI显示器,
# hilens studio中VideoCapture如果不填写参数，则默认读取test/camera0.mp4文件，
# 在hilens kit中不填写参数则读取本地摄像头
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
if rec:
rec_video(camera, display, show)
# 初始化模型
# -*- coding: utf-8 -*-
# model_path = os.path.join(work_path, 'model/yolo3_darknet53_raw3_4_sup_slope_terminal_t.om')
model_path = os.path.join(work_path, 'model/yolo3_darknet53_raw3_4_sup_slope_now_terminal_t.om')
driving_model = hilens.Model(model_path)
frame_index = 0
json_bbox_list = []
json_data = {'info': 'det_result'}
while True:
frame_index += 1
try:
time_start = time.time()
# 1. 设备接入 #####
input_yuv = camera.read() # 读取一帧图片(YUV NV21格式)
# 2. 数据预处理 #####
if rgb:
img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2RGB_NV21) # 转为RGB格式
else:
img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2BGR_NV21) # 转为BGR格式
if pad:
img_preprocess, img_w, img_h, new_w, new_h, shift_x_ratio, shift_y_ratio = preprocess_with_pad(img_rgb) # 缩放为模型输入尺寸
# 3. 模型推理 #####
output = driving_model.infer([img_preprocess.flatten()])
# 4. 获取检测结果 #####
bboxes = get_result_with_pad(output, img_w, img_h, new_w, new_h, shift_x_ratio, shift_y_ratio)
else:
img_preprocess, img_w, img_h = preprocess(img_rgb) # 缩放为模型输入尺寸
# 3. 模型推理 #####
output = driving_model.infer([img_preprocess.flatten()])
# 4. 获取检测结果 #####
bboxes = get_result(output, img_w, img_h)
# # 5-1. [比赛提交作品用] 将结果输出到json文件中 #####
# if len(bboxes) > 0:
# json_bbox = convert_to_json(bboxes, frame_index)
# json_bbox_list.append(json_bbox)
# # if bboxes != []:
# # print()
if socket_use:
data = data_generate_4(bboxes)
# 5-2. [调试用] 将结果输出到display #####
if show:
if rgb:
img_bgr = cv2.cvtColor(img_rgb, cv2.COLOR_RGB2BGR)
else:
img_bgr = img_rgb
img_bgr, labelName = draw_boxes(img_bgr, bboxes) # 在图像上画框
output_yuv = hilens.cvt_color(img_bgr, hilens.BGR2YUV_NV21)
display.show(output_yuv) # 显示到屏幕上
if log:
time_frame = 1000 * (time.time() - time_start)
hilens.info('----- time_frame = %.2fms -----' % time_frame)
except RuntimeError:
print('last frame')
break
# 保存检测结果
hilens.info('write json result to file')
result_filename = './result.json'
json_data['result'] = json_bbox_list
save_json_to_file(json_data, result_filename)
hilens.terminate()
