def run():
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("mask")
# 初始化摄像头
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
# 初始化模型
mask_model_path = hilens.get_model_dir() + "convert-mask-detection.om"
mask_model = hilens.Model(mask_model_path)
while True:
##### 1. 设备接入 #####
input_yuv = camera.read() # 读取一帧图片(YUV NV21格式)
##### 2. 数据预处理 #####
img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2RGB_NV21) # 转为RGB格式
img_preprocess, img_w, img_h = preprocess(img_rgb) # 缩放为模型输入尺寸
##### 3. 模型推理 #####
output = mask_model.infer([img_preprocess.flatten()])
##### 4. 结果输出 #####
bboxes = get_result(output, img_w, img_h) # 获取检测结果
img_rgb = draw_boxes(img_rgb, bboxes) # 在图像上画框
output_yuv = hilens.cvt_color(img_rgb, hilens.RGB2YUV_NV21)
display.show(output_yuv) # 显示到屏幕上
hilens.terminate()
def run_inner(work_path):
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("heart")
# 初始化摄像头与显示器
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
# 初始化模型
model_path = os.path.join(work_path, 'model/convert-6ecb.om')
cls_model = hilens.Model(model_path)
while True:
try:
# 2.1 读取摄像头数据
input_nv21 = camera.read()
# 2.2 截取出一个正方形区域作为手势识别输入并做预处理
input_rgb, gesture_area = preprocess(input_nv21)
input_resized = cv2.resize(gesture_area, (net_size, net_size))
# 2.3 模型推理
outputs = cls_model.infer([input_resized.flatten()])
# 2.4 结果展示
process_predict_result(outputs, input_rgb)
output_nv21 = hilens.cvt_color(input_rgb, hilens.RGB2YUV_NV21)
display.show(output_nv21)
except Exception as e:
print(e)
break
def run():
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("pose")
# 初始化模型
pose_model_path = hilens.get_model_dir() + "pose_template_model.om"
pose_model = hilens.Model(pose_model_path)
# 初始化USB摄像头与HDMI显示器
camera = hilens.VideoCapture()
display_hdmi = hilens.Display(hilens.HDMI)
while True:
# 读取一帧图片(BGR格式)
input_yuv = camera.read()
# 图片预处理：转为BGR格式、裁剪/缩放为模型输入尺寸
input_bgr = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2BGR_NV21)
img_preprocess = preprocess(input_bgr)
# 模型推理
model_outputs = pose_model.infer([img_preprocess.flatten()])
# 从推理结果中解码出人体关键点并画在图像中
points = get_points(input_bgr, model_outputs)
img_data = draw_limbs(input_bgr, points)
# 输出处理后的图像到HDMI显示器，必须先转换成YUV NV21格式
output_nv21 = hilens.cvt_color(img_data, hilens.BGR2YUV_NV21)
display_hdmi.show(output_nv21)
hilens.terminate()
def run(work_path):
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("driving")
# 初始化自带摄像头与HDMI显示器,
# hilens studio中VideoCapture如果不填写参数，则默认读取test/camera0.mp4文件，
# 在hilens kit中不填写参数则读取本地摄像头
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
# 初始化模型
model_path = os.path.join(work_path, 'model/yolo3.om')
driving_model = hilens.Model(model_path)
frame_index = 0
json_bbox_list = []
json_data = {'info': 'det_result'}
while True:
frame_index += 1
try:
time_start = time.time()
# 1. 设备接入 #####
input_yuv = camera.read() # 读取一帧图片(YUV NV21格式)
# 2. 数据预处理 #####
img_bgr = cv2.cvtColor(input_yuv,
cv2.COLOR_YUV2BGR_NV21) # 转为BGR格式
img_preprocess, img_w, img_h = preprocess(img_bgr) # 缩放为模型输入尺寸
# 3. 模型推理 #####
output = driving_model.infer([img_preprocess.flatten()])
# 4. 获取检测结果 #####
bboxes = get_result(output, img_w, img_h)
# 5-1. [比赛提交作品用] 将结果输出到json文件中 #####
if len(bboxes) > 0:
json_bbox = convert_to_json(bboxes, frame_index)
json_bbox_list.append(json_bbox)
# 5-2. [调试用] 将结果输出到模拟器中 #####
img_bgr = draw_boxes(img_bgr, bboxes) # 在图像上画框
output_yuv = hilens.cvt_color(img_bgr, hilens.BGR2YUV_NV21)
display.show(output_yuv) # 显示到屏幕上
time_frame = 1000 * (time.time() - time_start)
hilens.info('----- time_frame = %.2fms -----' % time_frame)
except RuntimeError:
print('last frame')
break
# 保存检测结果
hilens.info('write json result to file')
result_filename = './result.json'
json_data['result'] = json_bbox_list
save_json_to_file(json_data, result_filename)
hilens.terminate()
def display(self, frame):
if self.HILENS == False:
cv2.imshow("results", frame)
else:
# resize image
frame = cv2.resize(frame, (1280, 720))
output_yuv = hilens.cvt_color(frame, hilens.RGB2YUV_NV21)
# frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV)
# self.dst_video_writer.write(resized)
self.disp.show(output_yuv)
def display(self, frame=None):
if self.HILENS == False:
# frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV)
# dim = (720, 520)
# # resize image
# resized = cv2.resize(frame, dim, interpolation = cv2.INTER_AREA)
# self.disp.show(resized)
# self.dst_video_writer.write(frame)
cv2.imshow("results", frame)
else:
# resize image
frame = cv2.resize(frame, (1280, 720))
output_yuv = hilens.cvt_color(frame, hilens.RGB2YUV_NV21)
# frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV)
# self.dst_video_writer.write(resized)
self.disp.show(output_yuv)
def main():
""" 利用SkillFramework进行人脸检测模型的推理 """
hilens.init("test") # 参数要与创建技能时填写的检验值保持一致
model_path = hilens.get_model_dir() + "face_detection_demo.om" # 模型路径
model = hilens.Model(model_path)
display_hdmi = hilens.Display(hilens.HDMI) # 图像通过hdmi输出到屏幕
camera = hilens.VideoCapture()
while True:
# 1. 读取摄像头输入（yuv nv21）
input_nv21 = camera.read()
# 2. 转为bgr
input_bgr = cv2.cvtColor(input_nv21, cv2.COLOR_YUV2BGR_NV21)
src_image_height = input_bgr.shape[0]
src_image_width = input_bgr.shape[1]
# 3. 保留原图比例的resize为网络输入尺寸
im_scale1 = float(input_width) / float(src_image_width)
im_scale2 = float(input_height) / float(src_image_height)
im_scale = min(im_scale1, im_scale2)
input_bgr_rescaled = cv2.resize(input_bgr, None, None, fx=im_scale, fy=im_scale)
input_bgr_resized = np.zeros((input_height, input_width, 3), dtype = np.uint8)
input_bgr_resized[0:input_bgr_rescaled.shape[0],0:input_bgr_rescaled.shape[1],:] = input_bgr_rescaled
# 3. 推理
outputs = model.infer([input_bgr_resized.flatten()])
# 4. 后处理得到人脸bounding box，恢复到原图比例，画人脸框
detect_boxes = im_detect_nms(outputs[0])
if len(detect_boxes) > 0:
for rect in detect_boxes:
left = max(rect[0] / im_scale, 0)
top = max(rect[1] / im_scale, 0)
right = min(rect[2] / im_scale, src_image_width)
bottom = min(rect[3] / im_scale, src_image_height)
cv2.rectangle(input_bgr, (int(left), int(top)), (int(right), int(bottom)), 255, 2)
# 5. 输出图像，必须是yuv nv21形式
output_nv21 = hilens.cvt_color(input_bgr, hilens.BGR2YUV_NV21)
display_hdmi.show(output_nv21)
def handler(a, b):
hilens.init("hello")
model = hilens.Model(model_path)
display_hdmi = hilens.Display(hilens.HDMI)
camera = hilens.VideoCapture()
while True:
input_nv21 = camera.read()
input_bgr = cv2.cvtColor(input_nv21,cv2.COLOR_YUV2BGR_NV21)
input_resized = cv2.resize(input_bgr, (net_w, net_h))
img_preprocess, img_w, img_h, new_w, new_h, shift_x_ratio, shift_y_ratio = preprocess(input_bgr, aipp_flag)
outputs = model.infer([input_resized.flatten()])
res = get_result(outputs, img_w, img_h, new_w, new_h, shift_x_ratio, shift_y_ratio)
img_data = draw_box_on_img(input_bgr, res)
output_nv21 = hilens.cvt_color(img_data, hilens.BGR2YUV_NV21)
display_hdmi.show(output_nv21)
def run(self):
global src_img_buff
global results_buff
print("run PictureProcessorThreadGuard")
try:
while not SHUTDOWN_SIG:
# print("pp thread running")
img_bgr = src_img_buff.get()
light_label = ""
model_label = ""
xmin = 0
xmax = 0
ymin = 0
ymax = 0
light_label, dst_img = self.__pp.detect(img_bgr)
if self.__model_enable and light_label == "":
model_label, pt1, pt2 = self.__model.inference(img_bgr)
if model_label != "":
xmin = pt1[0]
ymin = pt1[1]
xmax = pt2[0]
ymax = pt2[1]
result_label = model_label if light_label == "" else light_label
if self.__socket_enable:
results_buff.put((result_label, xmax, xmin, ymax, ymin))
if self.__display is not None:
import hilens
h, w, c = img_bgr.shape
cv2.putText(img_bgr, result_label,
(int(w / 2), int(h / 2)),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 0, 0), 2)
output_yuv = hilens.cvt_color(img_bgr, hilens.BGR2YUV_NV21)
self.__display.show(output_yuv) # 显示到屏幕上
finally:
print('pp ended')
def run():
# 配置系统日志级别
hilens.set_log_level(hilens.ERROR)
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("gesture")
# 初始化模型
gesture_model_path = hilens.get_model_dir() + "gesture_template_model.om"
gesture_model = hilens.Model(gesture_model_path)
# 初始化本地摄像头与HDMI显示器
camera = hilens.VideoCapture()
display_hdmi = hilens.Display(hilens.HDMI)
# 上一次上传OBS图片的时间与上传间隔
last_upload_time = 0
upload_duration = 5
# 读取技能配置
skill_cfg = hilens.get_skill_config()
if skill_cfg is None or 'server_url' not in skill_cfg:
hilens.error("server_url not configured")
return
while True:
# 读取一帧图片(YUV NV21格式)
input_yuv = camera.read()
# 图片预处理：转为RGB格式、缩放为模型输入尺寸
img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2RGB_NV21)
img_preprocess, img_w, img_h = preprocess(img_rgb)
# 模型推理
output = gesture_model.infer([img_preprocess.flatten()])
# 后处理得到手势所在区域与类别，并在RGB图中画框
bboxes = get_result(output, img_w, img_h)
img_rgb = draw_boxes(img_rgb, bboxes)
# 输出处理后的图像到HDMI显示器，必须先转回YUV NV21格式
output_yuv = hilens.cvt_color(img_rgb, hilens.RGB2YUV_NV21)
display_hdmi.show(output_yuv)
# 上传OK手势图片到OBS，为防止OBS数据存储过多，间隔一定的时间才上传图片
if time.time() - last_upload_time > upload_duration:
# 截取出OK手势图片(如果有的话)
img_OK = get_OK(img_rgb, bboxes)
if img_OK is not None:
# 上传OK手势图片到OBS，图片(用当前时间命名)需要先转为BGR格式并按照jpg格式编码
img_OK = cv2.cvtColor(img_OK, cv2.COLOR_RGB2BGR)
img_OK = cv2.imencode('.jpg', img_OK)[1]
filename = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime())
ret = hilens.upload_bufer(filename + "_OK.jpg", img_OK,
"write")
if ret != 0:
hilens.error("upload pic failed!")
return
last_upload_time = time.time()
# 以POST方式传输处理后的整张图片
try:
post_msg(skill_cfg['server_url'], img_rgb)
except Exception as e:
hilens.error("post data failed!")
print("Reason : ", e)
hilens.terminate()
def run():
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("landmarks")
# 初始化自带摄像头与HDMI显示器
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
# 初始化模型：人脸检测模型(centerface)、人脸68个关键点检测模型
centerface_model_path = hilens.get_model_dir() + "centerface_template_model.om"
centerface_model = hilens.Model(centerface_model_path)
landmark_model_path = hilens.get_model_dir() + "landmark68_template_model.om"
landmark_model = hilens.Model(landmark_model_path)
# 本段代码展示如何录制HiLens Kit摄像头拍摄的视频
fps = 10
size = (1280, 720)
format = cv2.VideoWriter_fourcc('M','J','P','G') # 注意视频格式
writer = cv2.VideoWriter("face.avi", format, fps, size)
# 待保存视频的起始帧数，可自行调节或加入更多逻辑
frame_count = 0
frame_start = 100
frame_end = 150
uploaded = False
while True:
# 读取一帧图片(YUV NV21格式)
input_yuv = camera.read()
# 图片预处理：转为RGB格式、缩放为模型输入尺寸
img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2RGB_NV21)
img_pre = preprocess(img_rgb)
img_h, img_w = img_rgb.shape[:2]
# 人脸检测模型推理，并进行后处理得到画面中最大的人脸检测框
output = centerface_model.infer([img_pre.flatten()])
face_box = get_largest_face_box(output, img_h, img_w)
# 画面中检测到有人脸且满足一定条件
if face_box is not None:
# 截取出人脸区域并做预处理
img_face = preprocess_landmark(img_rgb, face_box)
# 人脸关键点模型推理，得到68个人脸关键点
output2 = landmark_model.infer([img_face.flatten()])
landmarks = output2[0].reshape(68, 2)
# 将人脸框和人脸关键点画在RGB图中
img_rgb = draw_landmarks(img_rgb, face_box, landmarks)
# 输出处理后的图像到HDMI显示器，必须先转换成YUV NV21格式
output_nv21 = hilens.cvt_color(img_rgb, hilens.RGB2YUV_NV21)
display.show(output_nv21)
# 录制一段视频并发送到OBS中
if not uploaded:
frame_count += 1
if frame_count > frame_end: # 录制结束点
uploaded = True
writer.release() # 先保存在本地
ret = hilens.upload_file("face.avi", "face.avi", "write") # 发送到OBS中
if ret != 0:
hilens.error("upload file failed!")
return
elif frame_count > frame_start: # 录制开始点
# 注意写入的图片格式必须为BGR
writer.write(cv2.cvtColor(img_rgb, cv2.COLOR_RGB2BGR))
hilens.terminate()
def capVideo(cap, flag, upload_uri):
global input_height
global input_width
stable = False
num_stable_frames = 0
stable_frames_threshold = 5
lens_mtx = lens_mtxs[flag]
lens_dist = lens_dists[flag]
while True:
start = cv2.getTickCount()
frame = cap.read()
input_bgr = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_NV21)
input_bgr = lens_distortion_adjustment(input_bgr,
lens_mtx=lens_mtx,
lens_dist=lens_dist)
h, w, _ = input_bgr.shape
cropped_input_bgr = input_bgr[h // 5:h // 5 * 4, w // 5:w // 5 * 4, :]
input_resized = cv2.resize(cropped_input_bgr,
(input_width, input_height))
res = predict(cropped_input_bgr, input_resized)
finish = cv2.getTickCount()
duration = (finish - start) / cv2.getTickFrequency()
fps = int(1 / duration)
output_nv21 = None
if len(res) != 0:
print('camera{} got res: {}'.format(flag, res))
deltaY = h // 5
deltaX = w // 5
for item in res:
item[0] += deltaX
item[2] += deltaX
item[1] += deltaY
item[3] += deltaY
num_stable_frames += 1
stable = num_stable_frames >= stable_frames_threshold
c = (0, 0, 255)
if stable:
c = (255, 0, 0)
img_data = draw_box_on_img(input_bgr, res, c)
output_nv21 = hilens.cvt_color(img_data, hilens.BGR2YUV_NV21)
else:
output_nv21 = frame
stable = False
num_stable_frames = 0
if flag == 1:
display_hdmi.show(output_nv21)
if stable:
print('stable, send data')
sendData(upload_uri, res)
sleep(3)
stable = False
num_stable_frames = 0
def run(work_path):
global data
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("driving")
# 初始化自带摄像头与HDMI显示器,
# hilens studio中VideoCapture如果不填写参数，则默认读取test/camera0.mp4文件，
# 在hilens kit中不填写参数则读取本地摄像头
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
if rec:
rec_video(camera, display, show)
# 初始化模型
# -*- coding: utf-8 -*-
# model_path = os.path.join(work_path, 'model/yolo3_darknet53_raw3_4_sup_slope_terminal_t.om')
model_path = os.path.join(work_path, 'model/yolo3_darknet53_raw3_4_sup_slope_now_terminal_t.om')
driving_model = hilens.Model(model_path)
frame_index = 0
json_bbox_list = []
json_data = {'info': 'det_result'}
while True:
frame_index += 1
try:
time_start = time.time()
# 1. 设备接入 #####
input_yuv = camera.read() # 读取一帧图片(YUV NV21格式)
# 2. 数据预处理 #####
if rgb:
img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2RGB_NV21) # 转为RGB格式
else:
img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_YUV2BGR_NV21) # 转为BGR格式
if pad:
img_preprocess, img_w, img_h, new_w, new_h, shift_x_ratio, shift_y_ratio = preprocess_with_pad(img_rgb) # 缩放为模型输入尺寸
# 3. 模型推理 #####
output = driving_model.infer([img_preprocess.flatten()])
# 4. 获取检测结果 #####
bboxes = get_result_with_pad(output, img_w, img_h, new_w, new_h, shift_x_ratio, shift_y_ratio)
else:
img_preprocess, img_w, img_h = preprocess(img_rgb) # 缩放为模型输入尺寸
# 3. 模型推理 #####
output = driving_model.infer([img_preprocess.flatten()])
# 4. 获取检测结果 #####
bboxes = get_result(output, img_w, img_h)
# # 5-1. [比赛提交作品用] 将结果输出到json文件中 #####
# if len(bboxes) > 0:
# json_bbox = convert_to_json(bboxes, frame_index)
# json_bbox_list.append(json_bbox)
# # if bboxes != []:
# # print()
if socket_use:
data = data_generate_4(bboxes)
# 5-2. [调试用] 将结果输出到display #####
if show:
if rgb:
img_bgr = cv2.cvtColor(img_rgb, cv2.COLOR_RGB2BGR)
else:
img_bgr = img_rgb
img_bgr, labelName = draw_boxes(img_bgr, bboxes) # 在图像上画框
output_yuv = hilens.cvt_color(img_bgr, hilens.BGR2YUV_NV21)
display.show(output_yuv) # 显示到屏幕上
if log:
time_frame = 1000 * (time.time() - time_start)
hilens.info('----- time_frame = %.2fms -----' % time_frame)
except RuntimeError:
print('last frame')
break
# 保存检测结果
hilens.info('write json result to file')
result_filename = './result.json'
json_data['result'] = json_bbox_list
save_json_to_file(json_data, result_filename)
hilens.terminate()
