def run_demo(args):
cap = open_images_capture(args.input, args.loop)
log.info('OpenVINO Runtime')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
log.info('Reading Object Detection model {}'.format(args.model_od))
detector_person = Detector(core, args.model_od,
device=args.device,
label_class=args.person_label)
log.info('The Object Detection model {} is loaded to {}'.format(args.model_od, args.device))
log.info('Reading Human Pose Estimation model {}'.format(args.model_hpe))
single_human_pose_estimator = HumanPoseEstimator(core, args.model_hpe,
device=args.device)
log.info('The Human Pose Estimation model {} is loaded to {}'.format(args.model_hpe, args.device))
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
video_writer = cv2.VideoWriter()
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 25)
metrics = PerformanceMetrics()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), (frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while frame is not None:
bboxes = detector_person.detect(frame)
human_poses = [single_human_pose_estimator.estimate(frame, bbox) for bbox in bboxes]
presenter.drawGraphs(frame)
colors = [(0, 0, 255),
(255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0),
(255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0),
(255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0),
(255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0)]
for pose, bbox in zip(human_poses, bboxes):
cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[0] + bbox[2], bbox[1] + bbox[3]), (255, 0, 0), 2)
for id_kpt, kpt in enumerate(pose):
cv2.circle(frame, (int(kpt[0]), int(kpt[1])), 3, colors[id_kpt], -1)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Human Pose Estimation Demo', frame)
key = cv2.waitKey(delay)
if key == 27:
break
presenter.handleKey(key)
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
next_frame_id = 1
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests,
model_parameters={'input_layouts': args.layout})
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
config = {
'target_size': args.tsize,
'aspect_ratio': frame.shape[1] / frame.shape[0],
'confidence_threshold': args.prob_threshold,
'padding_mode': 'center' if args.architecture_type == 'higherhrnet'
else None, # the 'higherhrnet' and 'ae' specific
'delta': 0.5 if args.architecture_type == 'higherhrnet' else
None, # the 'higherhrnet' and 'ae' specific
}
model = ImageModel.create_model(ARCHITECTURES[args.architecture_type],
model_adapter, config)
model.log_layers_info()
hpe_pipeline = AsyncPipeline(model)
hpe_pipeline.submit_data(frame, 0, {
'frame': frame,
'start_time': start_time
})
output_transform = OutputTransform(frame.shape[:2], args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
if args.output and not video_writer.open(args.output,
cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
while True:
if hpe_pipeline.callback_exceptions:
raise hpe_pipeline.callback_exceptions[0]
# Process all completed requests
results = hpe_pipeline.get_result(next_frame_id_to_show)
if results:
(poses, scores), frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(poses) and args.raw_output_message:
print_raw_results(poses, scores, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_poses(frame, poses, args.prob_threshold,
output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Pose estimation results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
continue
if hpe_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
break
# Submit for inference
hpe_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
hpe_pipeline.await_any()
hpe_pipeline.await_all()
if hpe_pipeline.callback_exceptions:
raise hpe_pipeline.callback_exceptions[0]
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = hpe_pipeline.get_result(next_frame_id_to_show)
(poses, scores), frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(poses) and args.raw_output_message:
print_raw_results(poses, scores, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_poses(frame, poses, args.prob_threshold, output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Pose estimation results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
hpe_pipeline.preprocess_metrics.get_latency(),
hpe_pipeline.inference_metrics.get_latency(),
hpe_pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
next_frame_id = 1
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
model = Deblurring(model_adapter, preload=False)
model.reshape(frame.shape)
model.log_layers_info()
pipeline = AsyncPipeline(model)
pipeline.submit_data(frame, 0, {'frame': frame, 'start_time': start_time})
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(frame.shape[1] / 4), round(frame.shape[0] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(2 * frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
break
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
result_frame, frame_meta = results
input_frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
if input_frame.shape != result_frame.shape:
input_frame = cv2.resize(
input_frame,
(result_frame.shape[1], result_frame.shape[0]))
final_image = cv2.hconcat([input_frame, result_frame])
render_metrics.update(rendering_start_time)
presenter.drawGraphs(final_image)
metrics.update(start_time, final_image)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(final_image)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Deblurring Results', final_image)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
result_frame, frame_meta = results
input_frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
if input_frame.shape != result_frame.shape:
input_frame = cv2.resize(
input_frame, (result_frame.shape[1], result_frame.shape[0]))
final_image = cv2.hconcat([input_frame, result_frame])
render_metrics.update(rendering_start_time)
presenter.drawGraphs(final_image)
metrics.update(start_time, final_image)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(final_image)
if not args.no_show:
cv2.imshow('Deblurring Results', final_image)
key = cv2.waitKey(1)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
pipeline.preprocess_metrics.get_latency(),
pipeline.inference_metrics.get_latency(),
pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
class_map = load_class_map(args.class_map)
if class_map is None:
raise RuntimeError("Can't read {}".format(args.class_map))
core = load_core()
person_detector = PersonDetector(args.detection_model,
args.device,
core,
num_requests=2,
output_shape=DETECTOR_OUTPUT_SHAPE)
action_recognizer = ActionRecognizer(args.action_model,
args.device,
core,
num_requests=2,
img_scale=ACTION_IMAGE_SCALE,
num_classes=len(class_map))
person_tracker = Tracker(person_detector, TRACKER_SCORE_THRESHOLD,
TRACKER_IOU_THRESHOLD)
video_stream = VideoStream(args.input, ACTION_NET_INPUT_FPS,
action_recognizer.input_length)
video_stream.start()
metrics = PerformanceMetrics()
visualizer = Visualizer(VISUALIZER_TRG_FPS)
visualizer.register_window('Demo')
presenter = monitors.Presenter(args.utilization_monitors)
samples_library = None
if args.samples_dir is not None and os.path.exists(args.samples_dir):
visualizer.register_window('Gesture library')
visualizer.start()
library_queue = visualizer.get_queue('Gesture library')
samples_library = VideoLibrary(args.samples_dir,
SAMPLES_MAX_WINDOW_SIZE,
list(class_map.values()), library_queue,
SAMPLES_TRG_FPS)
samples_library.start()
else:
visualizer.start()
last_caption = None
active_object_id = -1
tracker_labels_map = {}
tracker_labels = set()
frames_processed = 0
while True:
start_time = perf_counter()
frame = video_stream.get_live_frame()
batch = video_stream.get_batch()
if frame is None or batch is None:
break
if frames_processed == 0:
video_writer = cv2.VideoWriter()
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
video_stream.fps(), (frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
detections, tracker_labels_map = person_tracker.add_frame(
frame, len(OBJECT_IDS), tracker_labels_map)
if detections is None:
active_object_id = -1
last_caption = None
if len(detections) == 1:
active_object_id = 0
if active_object_id >= 0:
cur_det = [det for det in detections if det.id == active_object_id]
if len(cur_det) != 1:
active_object_id = -1
last_caption = None
continue
recognizer_result = action_recognizer(batch,
cur_det[0].roi.reshape(-1))
if recognizer_result is not None:
action_class_id = np.argmax(recognizer_result)
action_class_label = \
class_map[action_class_id] if class_map is not None else action_class_id
action_class_score = np.max(recognizer_result)
if action_class_score > args.action_threshold:
last_caption = 'Last gesture: {} '.format(
action_class_label)
presenter.drawGraphs(frame)
if detections is not None:
tracker_labels = {det.id for det in detections}
for det in detections:
roi_color = (0, 255,
0) if active_object_id == det.id else (128, 128,
128)
border_width = 2 if active_object_id == det.id else 1
person_roi = det.roi[0]
cv2.rectangle(frame, (person_roi[0], person_roi[1]),
(person_roi[2], person_roi[3]), roi_color,
border_width)
cv2.putText(frame, str(det.id),
(person_roi[0] + 10, person_roi[1] + 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.8, roi_color, 2)
if last_caption is not None:
cv2.putText(frame, last_caption, (10, frame.shape[0] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if args.no_show:
continue
visualizer.put_queue(frame, 'Demo')
key = visualizer.get_key()
if key == 27: # esc
break
elif key == ord(' '): # space
active_object_id = -1
last_caption = None
elif key == 13: # enter
last_caption = None
elif key in OBJECT_IDS: # 0-9
local_bbox_id = int(chr(key))
if local_bbox_id in tracker_labels:
active_object_id = local_bbox_id
else:
presenter.handleKey(key)
if samples_library is not None:
if key == ord('f'): # forward
samples_library.next()
elif key == ord('b'): # backward
samples_library.prev()
if samples_library is not None:
samples_library.release()
visualizer.release()
video_stream.release()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
# Plugin initialization for specified device and load extensions library if specified
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
# Read IR
log.info('Reading Proposal model {}'.format(args.model_pnet))
p_net = core.read_model(args.model_pnet)
if len(p_net.inputs) != 1:
raise RuntimeError("Pnet supports only single input topologies")
if len(p_net.outputs) != 2:
raise RuntimeError("Pnet supports two output topologies")
log.info('Reading Refine model {}'.format(args.model_rnet))
r_net = core.read_model(args.model_rnet)
if len(r_net.inputs) != 1:
raise RuntimeError("Rnet supports only single input topologies")
if len(r_net.outputs) != 2:
raise RuntimeError("Rnet supports two output topologies")
log.info('Reading Output model {}'.format(args.model_onet))
o_net = core.read_model(args.model_onet)
if len(o_net.inputs) != 1:
raise RuntimeError("Onet supports only single input topologies")
if len(o_net.outputs) != 3:
raise RuntimeError("Onet supports three output topologies")
pnet_input_tensor_name = p_net.inputs[0].get_any_name()
rnet_input_tensor_name = r_net.inputs[0].get_any_name()
onet_input_tensor_name = o_net.inputs[0].get_any_name()
for node in p_net.outputs:
if node.shape[1] == 2:
pnet_cls_name = node.get_any_name()
elif node.shape[1] == 4:
pnet_roi_name = node.get_any_name()
else:
raise RuntimeError("Unsupported output layer for Pnet")
for node in r_net.outputs:
if node.shape[1] == 2:
rnet_cls_name = node.get_any_name()
elif node.shape[1] == 4:
rnet_roi_name = node.get_any_name()
else:
raise RuntimeError("Unsupported output layer for Rnet")
for node in o_net.outputs:
if node.shape[1] == 2:
onet_cls_name = node.get_any_name()
elif node.shape[1] == 4:
onet_roi_name = node.get_any_name()
elif node.shape[1] == 10:
onet_pts_name = node.get_any_name()
else:
raise RuntimeError("Unsupported output layer for Onet")
next_frame_id = 0
metrics = PerformanceMetrics()
presenter = None
video_writer = cv2.VideoWriter()
is_loaded_before = False
while True:
start_time = perf_counter()
origin_image = cap.read()
if origin_image is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(origin_image.shape[1] / 4), round(origin_image.shape[0] / 8)))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), (origin_image.shape[1], origin_image.shape[0])):
raise RuntimeError("Can't open video writer")
next_frame_id += 1
rgb_image = cv2.cvtColor(origin_image, cv2.COLOR_BGR2RGB)
oh, ow, _ = rgb_image.shape
scales = utils.calculate_scales(rgb_image)
# *************************************
# Pnet stage
# *************************************
pnet_res = []
for i, scale in enumerate(scales):
hs = int(oh*scale)
ws = int(ow*scale)
image = preprocess_image(rgb_image, ws, hs)
p_net.reshape({pnet_input_tensor_name: PartialShape([1, 3, ws, hs])}) # Change weidth and height of input blob
compiled_pnet = core.compile_model(p_net, args.device)
infer_request_pnet = compiled_pnet.create_infer_request()
if i == 0 and not is_loaded_before:
log.info("The Proposal model {} is loaded to {}".format(args.model_pnet, args.device))
infer_request_pnet.infer(inputs={pnet_input_tensor_name: image})
p_res = {name: infer_request_pnet.get_tensor(name).data[:] for name in {pnet_roi_name, pnet_cls_name}}
pnet_res.append(p_res)
image_num = len(scales)
rectangles = []
for i in range(image_num):
roi = pnet_res[i][pnet_roi_name]
cls = pnet_res[i][pnet_cls_name]
_, _, out_h, out_w = cls.shape
out_side = max(out_h, out_w)
rectangle = utils.detect_face_12net(cls[0][1], roi[0], out_side, 1/scales[i], ow, oh,
score_threshold[0], iou_threshold[0])
rectangles.extend(rectangle)
rectangles = utils.NMS(rectangles, iou_threshold[1], 'iou')
# Rnet stage
if len(rectangles) > 0:
r_net.reshape({rnet_input_tensor_name: PartialShape([len(rectangles), 3, 24, 24])}) # Change batch size of input blob
compiled_rnet = core.compile_model(r_net, args.device)
infer_request_rnet = compiled_rnet.create_infer_request()
if not is_loaded_before:
log.info("The Refine model {} is loaded to {}".format(args.model_rnet, args.device))
rnet_input = []
for rectangle in rectangles:
crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])]
crop_img = preprocess_image(crop_img, 24, 24)
rnet_input.extend(crop_img)
infer_request_rnet.infer(inputs={rnet_input_tensor_name: rnet_input})
rnet_res = {name: infer_request_rnet.get_tensor(name).data[:] for name in {rnet_roi_name, rnet_cls_name}}
roi = rnet_res[rnet_roi_name]
cls = rnet_res[rnet_cls_name]
rectangles = utils.filter_face_24net(cls, roi, rectangles, ow, oh, score_threshold[1], iou_threshold[2])
# Onet stage
if len(rectangles) > 0:
o_net.reshape({onet_input_tensor_name: PartialShape([len(rectangles), 3, 48, 48])}) # Change batch size of input blob
compiled_onet = core.compile_model(o_net, args.device)
infer_request_onet = compiled_onet.create_infer_request()
if not is_loaded_before:
log.info("The Output model {} is loaded to {}".format(args.model_onet, args.device))
is_loaded_before = True
onet_input = []
for rectangle in rectangles:
crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])]
crop_img = preprocess_image(crop_img, 48, 48)
onet_input.extend(crop_img)
infer_request_onet.infer(inputs={onet_input_tensor_name: onet_input})
onet_res = {name: infer_request_onet.get_tensor(name).data[:] for name in {onet_roi_name, onet_cls_name, onet_pts_name}}
roi = onet_res[onet_roi_name]
cls = onet_res[onet_cls_name]
pts = onet_res[onet_pts_name]
rectangles = utils.filter_face_48net(cls, roi, pts, rectangles, ow, oh,
score_threshold[2], iou_threshold[3])
# display results
for rectangle in rectangles:
# Draw detected boxes
cv2.putText(origin_image, 'confidence: {:.2f}'.format(rectangle[4]),
(int(rectangle[0]), int(rectangle[1])), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0))
cv2.rectangle(origin_image, (int(rectangle[0]), int(rectangle[1])), (int(rectangle[2]), int(rectangle[3])),
(255, 0, 0), 1)
# Draw landmarks
for i in range(5, 15, 2):
cv2.circle(origin_image, (int(rectangle[i+0]), int(rectangle[i+1])), 2, (0, 255, 0))
metrics.update(start_time, origin_image)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id <= args.output_limit):
video_writer.write(origin_image)
if not args.no_show:
cv2.imshow('MTCNN Results', origin_image)
key = cv2.waitKey(1)
if key in {ord('q'), ord('Q'), 27}:
break
presenter.handleKey(key)
metrics.log_total()
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
frame_processor = FrameProcessor(args)
frame_num = 0
metrics = PerformanceMetrics()
presenter = None
output_transform = None
input_crop = None
if args.crop_size[0] > 0 and args.crop_size[1] > 0:
input_crop = np.array(args.crop_size)
elif not (args.crop_size[0] == 0 and args.crop_size[1] == 0):
raise ValueError('Both crop height and width should be positive')
video_writer = cv2.VideoWriter()
while True:
start_time = perf_counter()
frame = cap.read()
if frame is None:
if frame_num == 0:
raise ValueError("Can't read an image from the input")
break
if input_crop:
frame = center_crop(frame, input_crop)
if frame_num == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
output_resolution):
raise RuntimeError("Can't open video writer")
detections = frame_processor.process(frame)
presenter.drawGraphs(frame)
frame = draw_detections(frame, frame_processor, detections,
output_transform)
metrics.update(start_time, frame)
frame_num += 1
if video_writer.isOpened() and (args.output_limit <= 0
or frame_num <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Face recognition demo', frame)
key = cv2.waitKey(1)
# Quit
if key in {ord('q'), ord('Q'), 27}:
break
presenter.handleKey(key)
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config,
max_num_requests=args.num_infer_requests,
model_parameters={'input_layouts': args.layout})
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
configuration = {
'confidence_threshold': args.prob_threshold,
'path_to_labels': args.labels,
}
model = get_model(model_adapter, configuration)
model.log_layers_info()
pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
tracker = None
if not args.no_track and cap.get_type() in {'VIDEO', 'CAMERA'}:
tracker = StaticIOUTracker()
visualizer = InstanceSegmentationVisualizer(model.labels, args.show_boxes, args.show_scores)
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2], args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), tuple(output_resolution)):
raise RuntimeError("Can't open video writer")
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {'frame': frame, 'start_time': start_time})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
(scores, classes, boxes, masks), frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if args.raw_output_message:
print_raw_results(boxes, classes, scores, next_frame_id_to_show)
rendering_start_time = perf_counter()
masks_tracks_ids = tracker(masks, classes) if tracker else None
frame = visualizer(frame, boxes, classes, scores, masks, masks_tracks_ids)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Instance Segmentation results', frame)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
pipeline.await_all()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
(scores, classes, boxes, masks), frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if args.raw_output_message:
print_raw_results(boxes, classes, scores, next_frame_id_to_show)
rendering_start_time = perf_counter()
masks_tracks_ids = tracker(masks, classes) if tracker else None
frame = visualizer(frame, boxes, classes, scores, masks, masks_tracks_ids)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Instance Segmentation results', frame)
cv2.waitKey(1)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
pipeline.preprocess_metrics.get_latency(),
pipeline.inference_metrics.get_latency(),
pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def run(params, config, capture, detector, reid):
win_name = 'Multi camera tracking'
frame_number = 0
output_detections = [[] for _ in range(capture.get_num_sources())]
key = -1
if config.normalizer_config.enabled:
capture.add_transform(
NormalizerCLAHE(
config.normalizer_config.clip_limit,
config.normalizer_config.tile_size,
))
tracker = MultiCameraTracker(capture.get_num_sources(),
reid,
config.sct_config,
**vars(config.mct_config),
visual_analyze=config.analyzer)
thread_body = FramesThreadBody(capture,
max_queue_length=len(capture.captures) * 2)
frames_thread = Thread(target=thread_body)
frames_thread.start()
frames_read = False
set_output_params = False
prev_frames = thread_body.frames_queue.get()
detector.run_async(prev_frames, frame_number)
metrics = PerformanceMetrics()
presenter = monitors.Presenter(params.utilization_monitors, 0)
while thread_body.process:
if not params.no_show:
key = check_pressed_keys(key)
if key == 27:
break
presenter.handleKey(key)
start_time = time.perf_counter()
try:
frames = thread_body.frames_queue.get_nowait()
frames_read = True
except queue.Empty:
frames = None
if frames is None:
continue
all_detections = detector.wait_and_grab()
if params.save_detections:
update_detections(output_detections, all_detections, frame_number)
frame_number += 1
detector.run_async(frames, frame_number)
all_masks = [[] for _ in range(len(all_detections))]
for i, detections in enumerate(all_detections):
all_detections[i] = [det[0] for det in detections]
all_masks[i] = [det[2] for det in detections if len(det) == 3]
tracker.process(prev_frames, all_detections, all_masks)
tracked_objects = tracker.get_tracked_objects()
vis = visualize_multicam_detections(
prev_frames, tracked_objects, **vars(config.visualization_config))
metrics.update(start_time, vis)
presenter.drawGraphs(vis)
if not params.no_show:
cv.imshow(win_name, vis)
if frames_read and not set_output_params:
set_output_params = True
if len(params.output_video):
frame_size = [frame.shape[::-1] for frame in frames]
fps = capture.get_fps()
target_width, target_height = get_target_size(
frame_size, None, **vars(config.visualization_config))
video_output_size = (target_width, target_height)
fourcc = cv.VideoWriter_fourcc(*'XVID')
output_video = cv.VideoWriter(params.output_video, fourcc,
min(fps), video_output_size)
else:
output_video = None
if set_output_params and output_video:
output_video.write(cv.resize(vis, video_output_size))
prev_frames, frames = frames, prev_frames
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
thread_body.process = False
frames_thread.join()
if len(params.history_file):
save_json_file(params.history_file,
tracker.get_all_tracks_history(),
description='History file')
if len(params.save_detections):
save_json_file(params.save_detections,
output_detections,
description='Detections')
if len(config.embeddings.save_path):
save_embeddings(tracker.scts, **vars(config.embeddings))
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
model = MonoDepthModel(model_adapter)
model.log_layers_info()
pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2], args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {'start_time': start_time})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
depth_map, frame_meta = results
depth_map = apply_color_map(depth_map, output_transform)
start_time = frame_meta['start_time']
presenter.drawGraphs(depth_map)
metrics.update(start_time, depth_map)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(depth_map)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow(DEMO_NAME, depth_map)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
depth_map, frame_meta = results
depth_map = apply_color_map(depth_map, output_transform)
start_time = frame_meta['start_time']
presenter.drawGraphs(depth_map)
metrics.update(start_time, depth_map)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(depth_map)
if not args.no_show:
cv2.imshow(DEMO_NAME, depth_map)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
parser = argparse.ArgumentParser(description='Whiteboard inpainting demo')
parser.add_argument('-i', '--input', required=True,
help='Required. Path to a video file or a device node of a web-camera.')
parser.add_argument('--loop', default=False, action='store_true',
help='Optional. Enable reading the input in a loop.')
parser.add_argument('-o', '--output', required=False,
help='Optional. Name of the output file(s) to save.')
parser.add_argument('-limit', '--output_limit', required=False, default=1000, type=int,
help='Optional. Number of frames to store in output. '
'If 0 is set, all frames are stored.')
parser.add_argument('-m_i', '--m_instance_segmentation', type=str, required=False,
help='Required. Path to the instance segmentation model.')
parser.add_argument('-m_s', '--m_semantic_segmentation', type=str, required=False,
help='Required. Path to the semantic segmentation model.')
parser.add_argument('-t', '--threshold', type=float, default=0.6,
help='Optional. Threshold for person instance segmentation model.')
parser.add_argument('--no_show', help="Optional. Don't show output.", action='store_true')
parser.add_argument('-d', '--device', type=str, default='CPU',
help='Optional. Specify a target device to infer on. CPU, GPU, HDDL or MYRIAD is '
'acceptable. The demo will look for a suitable plugin for the device specified.')
parser.add_argument('-l', '--cpu_extension', type=str, default=None,
help='MKLDNN (CPU)-targeted custom layers. Absolute \
path to a shared library with the kernels impl.')
parser.add_argument('-u', '--utilization_monitors', default='', type=str,
help='Optional. List of monitors to show initially.')
args = parser.parse_args()
cap = open_images_capture(args.input, args.loop)
if cap.get_type() not in ('VIDEO', 'CAMERA'):
raise RuntimeError("The input should be a video file or a numeric camera ID")
if bool(args.m_instance_segmentation) == bool(args.m_semantic_segmentation):
raise ValueError('Set up exactly one of segmentation models: '
'--m_instance_segmentation or --m_semantic_segmentation')
labels_dir = Path(__file__).resolve().parents[3] / 'data/dataset_classes'
mouse = MouseClick()
if not args.no_show:
cv2.namedWindow(WINNAME)
cv2.setMouseCallback(WINNAME, mouse.get_points)
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
model_path = args.m_instance_segmentation if args.m_instance_segmentation else args.m_semantic_segmentation
log.info('Reading model {}'.format(model_path))
if args.m_instance_segmentation:
labels_file = str(labels_dir / 'coco_80cl_bkgr.txt')
segmentation = MaskRCNN(core, args.m_instance_segmentation, labels_file,
args.threshold, args.device, args.cpu_extension)
elif args.m_semantic_segmentation:
labels_file = str(labels_dir / 'cityscapes_19cl_bkgr.txt')
segmentation = SemanticSegmentation(core, args.m_semantic_segmentation, labels_file,
args.threshold, args.device, args.cpu_extension)
log.info('The model {} is loaded to {}'.format(model_path, args.device))
metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
black_board = False
frame_number = 0
key = -1
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
out_frame_size = (frame.shape[1], frame.shape[0] * 2)
output_frame = np.full((frame.shape[0], frame.shape[1], 3), 255, dtype='uint8')
presenter = monitors.Presenter(args.utilization_monitors, 20,
(out_frame_size[0] // 4, out_frame_size[1] // 16))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), out_frame_size):
raise RuntimeError("Can't open video writer")
while frame is not None:
mask = None
detections = segmentation.get_detections([frame])
expand_mask(detections, frame.shape[1] // 27)
if len(detections[0]) > 0:
mask = detections[0][0][2]
for i in range(1, len(detections[0])):
mask = cv2.bitwise_or(mask, detections[0][i][2])
if mask is not None:
mask = np.stack([mask, mask, mask], axis=-1)
else:
mask = np.zeros(frame.shape, dtype='uint8')
clear_frame = remove_background(frame, invert_colors=not black_board)
output_frame = np.where(mask, output_frame, clear_frame)
merged_frame = np.vstack([frame, output_frame])
merged_frame = cv2.resize(merged_frame, out_frame_size)
metrics.update(start_time, merged_frame)
if video_writer.isOpened() and (args.output_limit <= 0 or frame_number <= args.output_limit-1):
video_writer.write(merged_frame)
presenter.drawGraphs(merged_frame)
if not args.no_show:
cv2.imshow(WINNAME, merged_frame)
key = check_pressed_keys(key)
if key == 27: # 'Esc'
break
if key == ord('i'): # catch pressing of key 'i'
black_board = not black_board
output_frame = 255 - output_frame
else:
presenter.handleKey(key)
if mouse.crop_available:
x0, x1 = min(mouse.points[0][0], mouse.points[1][0]), \
max(mouse.points[0][0], mouse.points[1][0])
y0, y1 = min(mouse.points[0][1], mouse.points[1][1]), \
max(mouse.points[0][1], mouse.points[1][1])
x1, y1 = min(x1, output_frame.shape[1] - 1), min(y1, output_frame.shape[0] - 1)
board = output_frame[y0: y1, x0: x1, :]
if board.shape[0] > 0 and board.shape[1] > 0:
cv2.namedWindow('Board', cv2.WINDOW_KEEPRATIO)
cv2.imshow('Board', board)
frame_number += 1
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
if args.architecture_type != 'yolov4' and args.anchors:
log.warning(
'The "--anchors" option works only for "-at==yolov4". Option will be omitted'
)
if args.architecture_type != 'yolov4' and args.masks:
log.warning(
'The "--masks" option works only for "-at==yolov4". Option will be omitted'
)
if args.architecture_type not in ['nanodet', 'nanodet-plus'
] and args.num_classes:
log.warning(
'The "--num_classes" option works only for "-at==nanodet" and "-at==nanodet-plus". Option will be omitted'
)
cap = open_images_capture(args.input, args.loop)
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests,
model_parameters={'input_layouts': args.layout})
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
configuration = {
'resize_type': args.resize_type,
'mean_values': args.mean_values,
'scale_values': args.scale_values,
'reverse_input_channels': args.reverse_input_channels,
'path_to_labels': args.labels,
'confidence_threshold': args.prob_threshold,
'input_size': args.input_size, # The CTPN specific
'num_classes':
args.num_classes, # The NanoDet and NanoDetPlus specific
}
model = DetectionModel.create_model(args.architecture_type, model_adapter,
configuration)
model.log_layers_info()
detector_pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
palette = ColorPalette(len(model.labels) if model.labels else 100)
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
while True:
if detector_pipeline.callback_exceptions:
raise detector_pipeline.callback_exceptions[0]
# Process all completed requests
results = detector_pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(objects) and args.raw_output_message:
print_raw_results(objects, model.labels, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_detections(frame, objects, palette, model.labels,
output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Detection Results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
continue
if detector_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
detector_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
detector_pipeline.await_any()
detector_pipeline.await_all()
if detector_pipeline.callback_exceptions:
raise detector_pipeline.callback_exceptions[0]
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = detector_pipeline.get_result(next_frame_id_to_show)
objects, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(objects) and args.raw_output_message:
print_raw_results(objects, model.labels, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_detections(frame, objects, palette, model.labels,
output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Detection Results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
detector_pipeline.preprocess_metrics.get_latency(),
detector_pipeline.inference_metrics.get_latency(),
detector_pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
place_recognition = PlaceRecognition(args.model, args.device,
args.gallery_folder,
args.gallery_size)
compute_embeddings_times = []
search_in_gallery_times = []
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 0)
video_writer = cv2.VideoWriter()
metrics = PerformanceMetrics()
while True:
start_time = perf_counter()
frame = cap.read()
if frame is None:
if frames_processed == 0:
raise ValueError("Can't read an image from the input")
break
elapsed, probe_embedding = time_elapsed(
place_recognition.compute_embedding, frame)
compute_embeddings_times.append(elapsed)
elapsed, (sorted_indexes, distances) = time_elapsed(
place_recognition.search_in_gallery, probe_embedding)
search_in_gallery_times.append(elapsed)
image, key = visualize(
frame, [str(place_recognition.impaths[i]) for i in sorted_indexes],
distances[sorted_indexes],
place_recognition.input_size,
np.mean(compute_embeddings_times),
np.mean(search_in_gallery_times),
imshow_delay=3,
presenter=presenter,
no_show=args.no_show)
metrics.update(start_time)
if frames_processed == 0:
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(image.shape[1], image.shape[0])):
raise RuntimeError("Can't open video writer")
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(image)
if key == 27:
break
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
with open(args.labels, 'rt') as labels_file:
class_labels = labels_file.read().splitlines()
assert len(class_labels), 'The file with class labels is empty'
# Plugin initialization for specified device and load extensions library if specified.
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
if args.cpu_extension and 'CPU' in args.device:
core.add_extension(args.cpu_extension, 'CPU')
# Read IR
log.info('Reading model {}'.format(args.model))
model = core.read_model(args.model)
image_input, image_info_input, (
n, c, h,
w), model_type, output_names, postprocessor = check_model(model)
args.no_keep_aspect_ratio = model_type == 'yolact' or args.no_keep_aspect_ratio
compiled_model = core.compile_model(model, args.device)
infer_request = compiled_model.create_infer_request()
log.info('The model {} is loaded to {}'.format(args.model, args.device))
if args.no_track:
tracker = None
else:
tracker = StaticIOUTracker()
if args.delay:
delay = args.delay
else:
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
frames_processed = 0
metrics = PerformanceMetrics()
visualizer = Visualizer(class_labels,
show_boxes=args.show_boxes,
show_scores=args.show_scores)
video_writer = cv2.VideoWriter()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
out_frame_size = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 45,
(round(out_frame_size[0] / 4), round(out_frame_size[1] / 8)))
if args.output and not video_writer.open(args.output,
cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), out_frame_size):
raise RuntimeError("Can't open video writer")
while frame is not None:
if args.no_keep_aspect_ratio:
# Resize the image to a target size.
scale_x = w / frame.shape[1]
scale_y = h / frame.shape[0]
input_image = cv2.resize(frame, (w, h))
else:
# Resize the image to keep the same aspect ratio and to fit it to a window of a target size.
scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1])
input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y)
input_image_size = input_image.shape[:2]
input_image = np.pad(input_image,
((0, h - input_image_size[0]),
(0, w - input_image_size[1]), (0, 0)),
mode='constant',
constant_values=0)
# Change data layout from HWC to CHW.
input_image = input_image.transpose((2, 0, 1))
input_image = input_image.reshape((n, c, h, w)).astype(np.float32)
input_image_info = np.asarray(
[[input_image_size[0], input_image_size[1], 1]], dtype=np.float32)
# Run the model.
feed_dict = {image_input: input_image}
if image_info_input:
feed_dict[image_info_input] = input_image_info
infer_request.infer(feed_dict)
outputs = {
name: infer_request.get_tensor(name).data[:]
for name in output_names
}
# Parse detection results of the current request
scores, classes, boxes, masks = postprocessor(outputs, scale_x,
scale_y,
*frame.shape[:2], h, w,
args.prob_threshold)
if len(boxes) and args.raw_output_message:
log.debug(
' -------------------------- Frame # {} -------------------------- '
.format(frames_processed))
log.debug(
' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX '
)
for box, cls, score in zip(boxes, classes, scores):
log.debug(
'{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} '
.format(cls, score, *box))
# Get instance track IDs.
masks_tracks_ids = None
if tracker is not None:
masks_tracks_ids = tracker(masks, classes)
# Visualize masks.
frame = visualizer(frame, boxes, classes, scores, presenter, masks,
masks_tracks_ids)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
# Show resulting image.
cv2.imshow('Results', frame)
if not args.no_show:
key = cv2.waitKey(delay)
esc_code = 27
if key == esc_code:
break
presenter.handleKey(key)
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
delay = int(cap.get_type() in {'VIDEO', 'CAMERA'})
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
config = {
'mean_values': args.mean_values,
'scale_values': args.scale_values,
'reverse_input_channels': args.reverse_input_channels,
'topk': args.topk,
'path_to_labels': args.labels
}
model = Classification(model_adapter, config)
model.log_layers_info()
async_pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
ESC_KEY = 27
key = -1
while True:
if async_pipeline.callback_exceptions:
raise async_pipeline.callback_exceptions[0]
# Process all completed requests
results = async_pipeline.get_result(next_frame_id_to_show)
if results:
classifications, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if args.raw_output_message:
print_raw_results(classifications, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_labels(frame, classifications, output_transform)
if delay or args.no_show:
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Classification Results', frame)
key = cv2.waitKey(delay)
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
continue
if async_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
async_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
async_pipeline.await_any()
async_pipeline.await_all()
if key not in {ord('q'), ord('Q'), ESC_KEY}:
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show,
next_frame_id):
results = async_pipeline.get_result(next_frame_id_to_show)
while results is None:
results = async_pipeline.get_result(next_frame_id_to_show)
classifications, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if args.raw_output_message:
print_raw_results(classifications, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_labels(frame, classifications, output_transform)
if delay or args.no_show:
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Classification Results', frame)
key = cv2.waitKey(delay)
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
if delay or args.no_show:
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
async_pipeline.preprocess_metrics.get_latency(),
async_pipeline.inference_metrics.get_latency(),
async_pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
if cap.get_type() not in ('VIDEO', 'CAMERA'):
raise RuntimeError(
"The input should be a video file or a numeric camera ID")
frames = RoiDetectorOnVideo(cap)
img_retrieval = ImageRetrieval(args.model, args.device, args.gallery,
INPUT_SIZE, args.cpu_extension)
compute_embeddings_times = []
search_in_gallery_times = []
positions = []
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 0)
video_writer = cv2.VideoWriter()
metrics = PerformanceMetrics()
for image, view_frame in frames:
start_time = perf_counter()
position = None
sorted_indexes = []
if image is not None:
image = central_crop(image, divide_by=5, shift=1)
elapsed, probe_embedding = time_elapsed(
img_retrieval.compute_embedding, image)
compute_embeddings_times.append(elapsed)
elapsed, (sorted_indexes, distances) = time_elapsed(
img_retrieval.search_in_gallery, probe_embedding)
search_in_gallery_times.append(elapsed)
sorted_classes = [
img_retrieval.gallery_classes[i] for i in sorted_indexes
]
if args.ground_truth is not None:
position = sorted_classes.index(
img_retrieval.text_label_to_class_id[args.ground_truth])
positions.append(position)
log.info("ROI detected, found: %d, position of target: %d",
sorted_classes[0], position)
else:
log.info("ROI detected, found: %s", sorted_classes[0])
image, key = visualize(
view_frame,
position, [img_retrieval.impaths[i] for i in sorted_indexes],
distances[sorted_indexes] if position is not None else None,
img_retrieval.input_size,
np.mean(compute_embeddings_times),
np.mean(search_in_gallery_times),
imshow_delay=3,
presenter=presenter,
no_show=args.no_show)
metrics.update(start_time)
if frames_processed == 0:
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(image.shape[1], image.shape[0])):
raise RuntimeError("Can't open video writer")
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(image)
if key == 27:
break
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
if positions:
compute_metrics(positions)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
target_bgr = open_images_capture(args.target_bgr, loop=True) if args.target_bgr else None
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
labels = ['__background__', 'person'] if args.labels is None else load_labels(args.labels)
assert len(labels), 'The file with class labels is empty'
configuration = {
'confidence_threshold': args.prob_threshold,
'resize_type': args.resize_type
}
model, need_bgr_input = get_model(model_adapter, configuration, args)
input_bgr = open_images_capture(args.background, False).read() if need_bgr_input else None
person_id = -1
for i, label in enumerate(labels):
if label == 'person':
person_id = i
break
assert person_id >= 0, 'Person class did not find in labels list.'
model.log_layers_info()
pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
bgr = target_bgr.read() if target_bgr is not None else None
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2], args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), tuple(output_resolution)):
raise RuntimeError("Can't open video writer")
# Submit for inference
data = {'src': frame, 'bgr': input_bgr} if input_bgr is not None else frame
pipeline.submit_data(data, next_frame_id, {'frame': frame, 'start_time': start_time})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_results(frame, objects, output_resolution, bgr, person_id,
args.blur_bgr, args.show_with_original_frame)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Background subtraction results', frame)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show, model.labels)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_results(frame, objects, output_resolution, bgr, person_id,
args.blur_bgr, args.show_with_original_frame)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Background subtraction results', frame)
cv2.waitKey(1)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
pipeline.preprocess_metrics.get_latency(),
pipeline.inference_metrics.get_latency(),
pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
R = np.array(extrinsics['R'], dtype=np.float32)
t = np.array(extrinsics['t'], dtype=np.float32)
is_video = cap.get_type() in ('VIDEO', 'CAMERA')
base_height = args.height_size
fx = args.fx
frames_processed = 0
delay = 1
esc_code = 27
p_code = 112
space_code = 32
mean_time = 0
presenter = monitors.Presenter(args.utilization_monitors, 0)
metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while frame is not None:
current_time = cv2.getTickCount()
input_scale = base_height / frame.shape[0]
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests,
model_parameters={'input_layouts': args.layout})
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
model = SegmentationModel.create_model(args.architecture_type,
model_adapter,
{'path_to_labels': args.labels})
if args.architecture_type == 'segmentation':
visualizer = SegmentationVisualizer(args.colors)
if args.architecture_type == 'salient_object_detection':
visualizer = SaliencyMapVisualizer()
model.log_layers_info()
pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
only_masks = args.only_masks
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show, model.labels)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_segmentation(frame, objects, visualizer,
output_transform, only_masks)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Segmentation Results', frame)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
if key == 9:
only_masks = not only_masks
presenter.handleKey(key)
pipeline.await_all()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show, model.labels)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_segmentation(frame, objects, visualizer,
output_transform, only_masks)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Segmentation Results', frame)
key = cv2.waitKey(1)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
pipeline.preprocess_metrics.get_latency(),
pipeline.inference_metrics.get_latency(),
pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main(args):
cap = open_images_capture(args.input, args.loop)
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
log.info('Reading model {}'.format(args.model))
model = core.read_model(args.model)
input_tensor_name = 'data_l'
input_shape = model.input(input_tensor_name).shape
assert input_shape[1] == 1, "Expected model input shape with 1 channel"
inputs = {}
for input in model.inputs:
inputs[input.get_any_name()] = np.zeros(input.shape)
assert len(model.outputs) == 1, "Expected number of outputs is equal 1"
compiled_model = core.compile_model(model, device_name=args.device)
output_tensor = compiled_model.outputs[0]
infer_request = compiled_model.create_infer_request()
log.info('The model {} is loaded to {}'.format(args.model, args.device))
_, _, h_in, w_in = input_shape
frames_processed = 0
imshow_size = (640, 480)
graph_size = (imshow_size[0] // 2, imshow_size[1] // 4)
presenter = monitors.Presenter(args.utilization_monitors,
imshow_size[1] * 2 - graph_size[1],
graph_size)
metrics = PerformanceMetrics()
video_writer = cv.VideoWriter()
if args.output and not video_writer.open(
args.output, cv.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(imshow_size[0] * 2, imshow_size[1] * 2)):
raise RuntimeError("Can't open video writer")
start_time = perf_counter()
original_frame = cap.read()
if original_frame is None:
raise RuntimeError("Can't read an image from the input")
while original_frame is not None:
(h_orig, w_orig) = original_frame.shape[:2]
if original_frame.shape[2] > 1:
frame = cv.cvtColor(cv.cvtColor(original_frame, cv.COLOR_BGR2GRAY),
cv.COLOR_GRAY2RGB)
else:
frame = cv.cvtColor(original_frame, cv.COLOR_GRAY2RGB)
img_rgb = frame.astype(np.float32) / 255
img_lab = cv.cvtColor(img_rgb, cv.COLOR_RGB2Lab)
img_l_rs = cv.resize(img_lab.copy(), (w_in, h_in))[:, :, 0]
inputs[input_tensor_name] = np.expand_dims(img_l_rs, axis=[0, 1])
res = infer_request.infer(inputs)[output_tensor]
update_res = np.squeeze(res)
out = update_res.transpose((1, 2, 0))
out = cv.resize(out, (w_orig, h_orig))
img_lab_out = np.concatenate((img_lab[:, :, 0][:, :, np.newaxis], out),
axis=2)
img_bgr_out = np.clip(cv.cvtColor(img_lab_out, cv.COLOR_Lab2BGR), 0, 1)
original_image = cv.resize(original_frame, imshow_size)
grayscale_image = cv.resize(frame, imshow_size)
colorize_image = (cv.resize(img_bgr_out, imshow_size) * 255).astype(
np.uint8)
lab_image = cv.resize(img_lab_out, imshow_size).astype(np.uint8)
original_image = cv.putText(original_image, 'Original', (25, 50),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2,
cv.LINE_AA)
grayscale_image = cv.putText(grayscale_image, 'Grayscale', (25, 50),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255),
2, cv.LINE_AA)
colorize_image = cv.putText(colorize_image, 'Colorize', (25, 50),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2,
cv.LINE_AA)
lab_image = cv.putText(lab_image, 'LAB interpretation', (25, 50),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2,
cv.LINE_AA)
ir_image = [
cv.hconcat([original_image, grayscale_image]),
cv.hconcat([lab_image, colorize_image])
]
final_image = cv.vconcat(ir_image)
metrics.update(start_time, final_image)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(final_image)
presenter.drawGraphs(final_image)
if not args.no_show:
cv.imshow('Colorization Demo', final_image)
key = cv.waitKey(1)
if key in {ord("q"), ord("Q"), 27}:
break
presenter.handleKey(key)
start_time = perf_counter()
original_frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
# Read IR
log.info('Reading Mask-RCNN model {}'.format(args.mask_rcnn_model))
mask_rcnn_model = core.read_model(args.mask_rcnn_model)
input_tensor_name = 'image'
try:
n, c, h, w = mask_rcnn_model.input(input_tensor_name).shape
if n != 1:
raise RuntimeError(
'Only batch 1 is supported by the demo application')
except RuntimeError:
raise RuntimeError(
'Demo supports only topologies with the following input tensor name: {}'
.format(input_tensor_name))
required_output_names = {'boxes', 'labels', 'masks', 'text_features'}
for output_tensor_name in required_output_names:
try:
mask_rcnn_model.output(output_tensor_name)
except RuntimeError:
raise RuntimeError(
'Demo supports only topologies with the following output tensor names: {}'
.format(', '.join(required_output_names)))
log.info('Reading Text Recognition Encoder model {}'.format(
args.text_enc_model))
text_enc_model = core.read_model(args.text_enc_model)
log.info('Reading Text Recognition Decoder model {}'.format(
args.text_dec_model))
text_dec_model = core.read_model(args.text_dec_model)
mask_rcnn_compiled_model = core.compile_model(mask_rcnn_model,
device_name=args.device)
mask_rcnn_infer_request = mask_rcnn_compiled_model.create_infer_request()
log.info('The Mask-RCNN model {} is loaded to {}'.format(
args.mask_rcnn_model, args.device))
text_enc_compiled_model = core.compile_model(text_enc_model, args.device)
text_enc_output_tensor = text_enc_compiled_model.outputs[0]
text_enc_infer_request = text_enc_compiled_model.create_infer_request()
log.info('The Text Recognition Encoder model {} is loaded to {}'.format(
args.text_enc_model, args.device))
text_dec_compiled_model = core.compile_model(text_dec_model, args.device)
text_dec_infer_request = text_dec_compiled_model.create_infer_request()
log.info('The Text Recognition Decoder model {} is loaded to {}'.format(
args.text_dec_model, args.device))
hidden_shape = text_dec_model.input(args.trd_input_prev_hidden).shape
text_dec_output_names = {
args.trd_output_symbols_distr, args.trd_output_cur_hidden
}
if args.no_track:
tracker = None
else:
tracker = StaticIOUTracker()
if args.delay:
delay = args.delay
else:
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
visualizer = InstanceSegmentationVisualizer(show_boxes=args.show_boxes,
show_scores=args.show_scores)
frames_processed = 0
metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
presenter = monitors.Presenter(args.utilization_monitors, 45,
(frame.shape[1] // 4, frame.shape[0] // 8))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while frame is not None:
if not args.keep_aspect_ratio:
# Resize the image to a target size.
scale_x = w / frame.shape[1]
scale_y = h / frame.shape[0]
input_image = cv2.resize(frame, (w, h))
else:
# Resize the image to keep the same aspect ratio and to fit it to a window of a target size.
scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1])
input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y)
input_image_size = input_image.shape[:2]
input_image = np.pad(input_image,
((0, h - input_image_size[0]),
(0, w - input_image_size[1]), (0, 0)),
mode='constant',
constant_values=0)
# Change data layout from HWC to CHW.
input_image = input_image.transpose((2, 0, 1))
input_image = input_image.reshape((n, c, h, w)).astype(np.float32)
# Run the MaskRCNN model.
mask_rcnn_infer_request.infer({input_tensor_name: input_image})
outputs = {
name: mask_rcnn_infer_request.get_tensor(name).data[:]
for name in required_output_names
}
# Parse detection results of the current request
boxes = outputs['boxes'][:, :4]
scores = outputs['boxes'][:, 4]
classes = outputs['labels'].astype(np.uint32)
raw_masks = outputs['masks']
text_features = outputs['text_features']
# Filter out detections with low confidence.
detections_filter = scores > args.prob_threshold
scores = scores[detections_filter]
classes = classes[detections_filter]
boxes = boxes[detections_filter]
raw_masks = raw_masks[detections_filter]
text_features = text_features[detections_filter]
boxes[:, 0::2] /= scale_x
boxes[:, 1::2] /= scale_y
masks = []
for box, cls, raw_mask in zip(boxes, classes, raw_masks):
mask = segm_postprocess(box, raw_mask, frame.shape[0],
frame.shape[1])
masks.append(mask)
texts = []
for feature in text_features:
input_data = {'input': np.expand_dims(feature, axis=0)}
feature = text_enc_infer_request.infer(
input_data)[text_enc_output_tensor]
feature = np.reshape(feature,
(feature.shape[0], feature.shape[1], -1))
feature = np.transpose(feature, (0, 2, 1))
hidden = np.zeros(hidden_shape)
prev_symbol_index = np.ones((1, )) * SOS_INDEX
text = ''
text_confidence = 1.0
for i in range(MAX_SEQ_LEN):
text_dec_infer_request.infer({
args.trd_input_prev_symbol:
np.reshape(prev_symbol_index, (1, )),
args.trd_input_prev_hidden:
hidden,
args.trd_input_encoder_outputs:
feature
})
decoder_output = {
name: text_dec_infer_request.get_tensor(name).data[:]
for name in text_dec_output_names
}
symbols_distr = decoder_output[args.trd_output_symbols_distr]
symbols_distr_softmaxed = softmax(symbols_distr, axis=1)[0]
prev_symbol_index = int(np.argmax(symbols_distr, axis=1))
text_confidence *= symbols_distr_softmaxed[prev_symbol_index]
if prev_symbol_index == EOS_INDEX:
break
text += args.alphabet[prev_symbol_index]
hidden = decoder_output[args.trd_output_cur_hidden]
texts.append(text if text_confidence >= args.tr_threshold else '')
if len(boxes) and args.raw_output_message:
log.debug(
' -------------------------- Frame # {} -------------------------- '
.format(frames_processed))
log.debug(
' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX '
)
for box, cls, score, mask in zip(boxes, classes, scores, masks):
log.debug(
'{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} '
.format(cls, score, *box))
# Get instance track IDs.
masks_tracks_ids = None
if tracker is not None:
masks_tracks_ids = tracker(masks, classes)
presenter.drawGraphs(frame)
# Visualize masks.
frame = visualizer(frame, boxes, classes, scores, masks,
masks_tracks_ids, texts)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
# Show resulting image.
cv2.imshow('Results', frame)
if not args.no_show:
key = cv2.waitKey(delay)
esc_code = 27
if key == esc_code:
break
presenter.handleKey(key)
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
