def __init__(self, input, loop):
self.loop = loop
self.reader_metrics = PerformanceMetrics()
self.cap = cv2.VideoCapture()
status = self.cap.open(input)
if not status:
raise InvalidInput("Can't open the video from {}".format(input))
class CameraCapWrapper(ImagesCapture):
def __init__(self, input, camera_resolution):
self.reader_metrics = PerformanceMetrics()
self.cap = cv2.VideoCapture()
try:
status = self.cap.open(int(input))
self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, camera_resolution[0])
self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, camera_resolution[1])
self.cap.set(cv2.CAP_PROP_FPS, 30)
self.cap.set(cv2.CAP_PROP_AUTOFOCUS, 1)
self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
if not status:
raise OpenError("Can't open the camera from {}".format(input))
except ValueError:
raise InvalidInput("Can't find the camera {}".format(input))
def read(self):
start_time = perf_counter()
status, image = self.cap.read()
if not status:
return None
self.reader_metrics.update(start_time)
return image
def fps(self):
return self.cap.get(cv2.CAP_PROP_FPS)
def get_type(self):
return 'CAMERA'
class VideoCapWrapper(ImagesCapture):
def __init__(self, input, loop):
self.loop = loop
self.reader_metrics = PerformanceMetrics()
self.cap = cv2.VideoCapture()
status = self.cap.open(input)
if not status:
raise InvalidInput("Can't open the video from {}".format(input))
def read(self):
start_time = perf_counter()
status, image = self.cap.read()
if not status:
if not self.loop:
return None
self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
status, image = self.cap.read()
if not status:
return None
self.reader_metrics.update(start_time)
return image
def fps(self):
return self.cap.get(cv2.CAP_PROP_FPS)
def get_type(self):
return 'VIDEO'
class ImreadWrapper(ImagesCapture):
def __init__(self, input, loop):
self.loop = loop
self.reader_metrics = PerformanceMetrics()
start_time = perf_counter()
if not os.path.isfile(input):
raise InvalidInput("Can't find the image by {}".format(input))
self.image = cv2.imread(input, cv2.IMREAD_COLOR)
if self.image is None:
raise OpenError("Can't open the image from {}".format(input))
self.reader_metrics.update(start_time)
self.can_read = True
def read(self):
if self.loop:
return copy.deepcopy(self.image)
if self.can_read:
self.can_read = False
return copy.deepcopy(self.image)
return None
def fps(self):
return 1.0
def get_type(self):
return 'IMAGE'
def __init__(self, input, loop):
self.loop = loop
self.reader_metrics = PerformanceMetrics()
start_time = perf_counter()
if not os.path.isfile(input):
raise InvalidInput("Can't find the image by {}".format(input))
self.image = cv2.imread(input, cv2.IMREAD_COLOR)
if self.image is None:
raise OpenError("Can't open the image from {}".format(input))
self.reader_metrics.update(start_time)
self.can_read = True
def __init__(self, input, camera_resolution):
self.reader_metrics = PerformanceMetrics()
self.cap = cv2.VideoCapture()
try:
status = self.cap.open(int(input))
self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, camera_resolution[0])
self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, camera_resolution[1])
self.cap.set(cv2.CAP_PROP_FPS, 30)
self.cap.set(cv2.CAP_PROP_AUTOFOCUS, 1)
self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
if not status:
raise OpenError("Can't open the camera from {}".format(input))
except ValueError:
raise InvalidInput("Can't find the camera {}".format(input))
def __init__(self, input, loop):
self.loop = loop
self.reader_metrics = PerformanceMetrics()
self.dir = input
if not os.path.isdir(self.dir):
raise InvalidInput("Can't find the dir by {}".format(input))
self.names = sorted(os.listdir(self.dir))
if not self.names:
raise OpenError("The dir {} is empty".format(input))
self.file_id = 0
for name in self.names:
filename = os.path.join(self.dir, name)
image = cv2.imread(filename, cv2.IMREAD_COLOR)
if image is not None:
return
raise OpenError("Can't read the first image from {}".format(input))
class DirReader(ImagesCapture):
def __init__(self, input, loop):
self.loop = loop
self.reader_metrics = PerformanceMetrics()
self.dir = input
if not os.path.isdir(self.dir):
raise InvalidInput("Can't find the dir by {}".format(input))
self.names = sorted(os.listdir(self.dir))
if not self.names:
raise OpenError("The dir {} is empty".format(input))
self.file_id = 0
for name in self.names:
filename = os.path.join(self.dir, name)
image = cv2.imread(filename, cv2.IMREAD_COLOR)
if image is not None:
return
raise OpenError("Can't read the first image from {}".format(input))
def read(self):
start_time = perf_counter()
while self.file_id < len(self.names):
filename = os.path.join(self.dir, self.names[self.file_id])
image = cv2.imread(filename, cv2.IMREAD_COLOR)
self.file_id += 1
if image is not None:
self.reader_metrics.update(start_time)
return image
if self.loop:
self.file_id = 0
while self.file_id < len(self.names):
filename = os.path.join(self.dir, self.names[self.file_id])
image = cv2.imread(filename, cv2.IMREAD_COLOR)
self.file_id += 1
if image is not None:
self.reader_metrics.update(start_time)
return image
return None
def fps(self):
return 1.0
def get_type(self):
return 'DIR'
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
# Plugin initialization for specified device and load extensions library if specified
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
ie = IECore()
# Read IR
log.info('Reading Proposal model {}'.format(args.model_pnet))
p_net = ie.read_network(args.model_pnet)
assert len(p_net.input_info.keys()) == 1, "Pnet supports only single input topologies"
assert len(p_net.outputs) == 2, "Pnet supports two output topologies"
log.info('Reading Refine model {}'.format(args.model_rnet))
r_net = ie.read_network(args.model_rnet)
assert len(r_net.input_info.keys()) == 1, "Rnet supports only single input topologies"
assert len(r_net.outputs) == 2, "Rnet supports two output topologies"
log.info('Reading Output model {}'.format(args.model_onet))
o_net = ie.read_network(args.model_onet)
assert len(o_net.input_info.keys()) == 1, "Onet supports only single input topologies"
assert len(o_net.outputs) == 3, "Onet supports three output topologies"
pnet_input_blob = next(iter(p_net.input_info))
rnet_input_blob = next(iter(r_net.input_info))
onet_input_blob = next(iter(o_net.input_info))
for name, blob in p_net.outputs.items():
if blob.shape[1] == 2:
pnet_cls_name = name
elif blob.shape[1] == 4:
pnet_roi_name = name
else:
raise RuntimeError("Unsupported output layer for Pnet")
for name, blob in r_net.outputs.items():
if blob.shape[1] == 2:
rnet_cls_name = name
elif blob.shape[1] == 4:
rnet_roi_name = name
else:
raise RuntimeError("Unsupported output layer for Rnet")
for name, blob in o_net.outputs.items():
if blob.shape[1] == 2:
onet_cls_name = name
elif blob.shape[1] == 4:
onet_roi_name = name
elif blob.shape[1] == 10:
onet_pts_name = name
else:
raise RuntimeError("Unsupported output layer for Onet")
next_frame_id = 0
metrics = PerformanceMetrics()
presenter = None
video_writer = cv2.VideoWriter()
is_loaded_before = False
while True:
start_time = perf_counter()
origin_image = cap.read()
if origin_image is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(origin_image.shape[1] / 4), round(origin_image.shape[0] / 8)))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), (origin_image.shape[1], origin_image.shape[0])):
raise RuntimeError("Can't open video writer")
next_frame_id += 1
rgb_image = cv2.cvtColor(origin_image, cv2.COLOR_BGR2RGB)
oh, ow, _ = rgb_image.shape
scales = utils.calculate_scales(rgb_image)
# *************************************
# Pnet stage
# *************************************
pnet_res = []
for i, scale in enumerate(scales):
hs = int(oh*scale)
ws = int(ow*scale)
image = preprocess_image(rgb_image, ws, hs)
p_net.reshape({pnet_input_blob: [1, 3, ws, hs]}) # Change weidth and height of input blob
exec_pnet = ie.load_network(network=p_net, device_name=args.device)
if i == 0 and not is_loaded_before:
log.info("The Proposal model {} is loaded to {}".format(args.model_pnet, args.device))
p_res = exec_pnet.infer(inputs={pnet_input_blob: image})
pnet_res.append(p_res)
image_num = len(scales)
rectangles = []
for i in range(image_num):
roi = pnet_res[i][pnet_roi_name]
cls = pnet_res[i][pnet_cls_name]
_, _, out_h, out_w = cls.shape
out_side = max(out_h, out_w)
rectangle = utils.detect_face_12net(cls[0][1], roi[0], out_side, 1/scales[i], ow, oh,
score_threshold[0], iou_threshold[0])
rectangles.extend(rectangle)
rectangles = utils.NMS(rectangles, iou_threshold[1], 'iou')
# Rnet stage
if len(rectangles) > 0:
r_net.reshape({rnet_input_blob: [len(rectangles), 3, 24, 24]}) # Change batch size of input blob
exec_rnet = ie.load_network(network=r_net, device_name=args.device)
if not is_loaded_before:
log.info("The Refine model {} is loaded to {}".format(args.model_rnet, args.device))
rnet_input = []
for rectangle in rectangles:
crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])]
crop_img = preprocess_image(crop_img, 24, 24)
rnet_input.extend(crop_img)
rnet_res = exec_rnet.infer(inputs={rnet_input_blob: rnet_input})
roi = rnet_res[rnet_roi_name]
cls = rnet_res[rnet_cls_name]
rectangles = utils.filter_face_24net(cls, roi, rectangles, ow, oh, score_threshold[1], iou_threshold[2])
# Onet stage
if len(rectangles) > 0:
o_net.reshape({onet_input_blob: [len(rectangles), 3, 48, 48]}) # Change batch size of input blob
exec_onet = ie.load_network(network=o_net, device_name=args.device)
if not is_loaded_before:
log.info("The Output model {} is loaded to {}".format(args.model_onet, args.device))
is_loaded_before = True
onet_input = []
for rectangle in rectangles:
crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])]
crop_img = preprocess_image(crop_img, 48, 48)
onet_input.extend(crop_img)
onet_res = exec_onet.infer(inputs={onet_input_blob: onet_input})
roi = onet_res[onet_roi_name]
cls = onet_res[onet_cls_name]
pts = onet_res[onet_pts_name]
rectangles = utils.filter_face_48net(cls, roi, pts, rectangles, ow, oh,
score_threshold[2], iou_threshold[3])
# display results
for rectangle in rectangles:
# Draw detected boxes
cv2.putText(origin_image, 'confidence: {:.2f}'.format(rectangle[4]),
(int(rectangle[0]), int(rectangle[1])), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0))
cv2.rectangle(origin_image, (int(rectangle[0]), int(rectangle[1])), (int(rectangle[2]), int(rectangle[3])),
(255, 0, 0), 1)
# Draw landmarks
for i in range(5, 15, 2):
cv2.circle(origin_image, (int(rectangle[i+0]), int(rectangle[i+1])), 2, (0, 255, 0))
metrics.update(start_time, origin_image)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id <= args.output_limit):
video_writer.write(origin_image)
if not args.no_show:
cv2.imshow('MTCNN Results', origin_image)
key = cv2.waitKey(1)
if key in {ord('q'), ord('Q'), 27}:
break
presenter.handleKey(key)
metrics.log_total()
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
next_frame_id = 1
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
config = {
'target_size': args.tsize,
'aspect_ratio': frame.shape[1] / frame.shape[0],
'prob_threshold': args.prob_threshold,
'padding_mode': 'center' if args.architecture_type == 'higherhrnet'
else None, # the 'higherhrnet' and 'ae' specific
'delta':
0.5 if 'higherhrnet' else None, # the 'higherhrnet' and 'ae' specific
}
model = ImageModel.create_model(ARCHITECTURES[args.architecture_type],
model_adapter, config)
model.log_layers_info()
hpe_pipeline = AsyncPipeline(model)
hpe_pipeline.submit_data(frame, 0, {
'frame': frame,
'start_time': start_time
})
output_transform = OutputTransform(frame.shape[:2], args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
if args.output and not video_writer.open(args.output,
cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
while True:
if hpe_pipeline.callback_exceptions:
raise hpe_pipeline.callback_exceptions[0]
# Process all completed requests
results = hpe_pipeline.get_result(next_frame_id_to_show)
if results:
(poses, scores), frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(poses) and args.raw_output_message:
print_raw_results(poses, scores, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_poses(frame, poses, args.prob_threshold,
output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Pose estimation results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
continue
if hpe_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
break
# Submit for inference
hpe_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
hpe_pipeline.await_any()
hpe_pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = hpe_pipeline.get_result(next_frame_id_to_show)
while results is None:
results = hpe_pipeline.get_result(next_frame_id_to_show)
(poses, scores), frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(poses) and args.raw_output_message:
print_raw_results(poses, scores, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_poses(frame, poses, args.prob_threshold, output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Pose estimation results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
hpe_pipeline.preprocess_metrics.get_latency(),
hpe_pipeline.inference_metrics.get_latency(),
hpe_pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
target_bgr = open_images_capture(args.target_bgr, loop=True) if args.target_bgr else None
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'remote':
log.info('Reading model {}'.format(args.model))
serving_config = {"address": "localhost", "port": 9000}
model_adapter = RemoteAdapter(args.model, serving_config)
labels = ['__background__', 'person'] if args.labels is None else args.labels
configuration = {
'prob_threshold': args.prob_threshold,
'resize_type': args.resize_type
}
model = get_model(model_adapter, configuration)
person_id = -1
for i, label in enumerate(labels):
if label == 'person':
person_id = i
break
assert person_id >= 0, 'Person class did not find in labels list.'
model.log_layers_info()
pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
bgr = target_bgr.read() if target_bgr is not None else None
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2], args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), tuple(output_resolution)):
raise RuntimeError("Can't open video writer")
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {'frame': frame, 'start_time': start_time})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_results(frame, objects, output_resolution, bgr, person_id,
args.blur_bgr, args.show_with_original_frame)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Background subtraction results', frame)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show, model.labels)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_results(frame, objects, output_resolution, bgr, person_id,
args.blur_bgr, args.show_with_original_frame)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Background subtraction results', frame)
cv2.waitKey(1)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
pipeline.preprocess_metrics.get_latency(),
pipeline.inference_metrics.get_latency(),
pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
if args.architecture_type != 'yolov4' and args.anchors:
log.warning(
'The "--anchors" options works only for "-at==yolov4". Option will be omitted'
)
if args.architecture_type != 'yolov4' and args.masks:
log.warning(
'The "--masks" options works only for "-at==yolov4". Option will be omitted'
)
cap = open_images_capture(args.input, args.loop)
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'remote':
log.info('Reading model {}'.format(args.model))
serving_config = {"address": "localhost", "port": 9000}
model_adapter = RemoteAdapter(args.model, serving_config)
configuration = {
'resize_type': args.resize_type,
'mean_values': args.mean_values,
'scale_values': args.scale_values,
'reverse_input_channels': args.reverse_input_channels,
'path_to_labels': args.labels,
'confidence_threshold': args.prob_threshold,
'input_size': args.input_size, # The CTPN specific
}
model = DetectionModel.create_model(args.architecture_type, model_adapter,
configuration)
model.log_layers_info()
detector_pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
palette = ColorPalette(len(model.labels) if model.labels else 100)
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
while True:
if detector_pipeline.callback_exceptions:
raise detector_pipeline.callback_exceptions[0]
# Process all completed requests
results = detector_pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(objects) and args.raw_output_message:
print_raw_results(objects, model.labels, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_detections(frame, objects, palette, model.labels,
output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Detection Results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
continue
if detector_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
detector_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
detector_pipeline.await_any()
detector_pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = detector_pipeline.get_result(next_frame_id_to_show)
while results is None:
results = detector_pipeline.get_result(next_frame_id_to_show)
objects, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(objects) and args.raw_output_message:
print_raw_results(objects, model.labels, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_detections(frame, objects, palette, model.labels,
output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Detection Results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
detector_pipeline.preprocess_metrics.get_latency(),
detector_pipeline.inference_metrics.get_latency(),
detector_pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'remote':
log.info('Reading model {}'.format(args.model))
serving_config = {"address": "localhost", "port": 9000}
model_adapter = RemoteAdapter(args.model, serving_config)
model = SegmentationModel.create_model(args.architecture_type,
model_adapter,
{'path_to_labels': args.labels})
if args.architecture_type == 'segmentation':
visualizer = SegmentationVisualizer(args.colors)
if args.architecture_type == 'salient_object_detection':
visualizer = SaliencyMapVisualizer()
model.log_layers_info()
pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
only_masks = args.only_masks
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show, model.labels)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_segmentation(frame, objects, visualizer,
output_transform, only_masks)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Segmentation Results', frame)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
if key == 9:
only_masks = not only_masks
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show, model.labels)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_segmentation(frame, objects, visualizer,
output_transform, only_masks)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Segmentation Results', frame)
key = cv2.waitKey(1)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
pipeline.preprocess_metrics.get_latency(),
pipeline.inference_metrics.get_latency(),
pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
if cap.get_type() not in ('VIDEO', 'CAMERA'):
raise RuntimeError("The input should be a video file or a numeric camera ID")
frames = RoiDetectorOnVideo(cap)
img_retrieval = ImageRetrieval(args.model, args.device, args.gallery, INPUT_SIZE,
args.cpu_extension)
compute_embeddings_times = []
search_in_gallery_times = []
positions = []
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 0)
video_writer = cv2.VideoWriter()
metrics = PerformanceMetrics()
for image, view_frame in frames:
start_time = perf_counter()
position = None
sorted_indexes = []
if image is not None:
image = central_crop(image, divide_by=5, shift=1)
elapsed, probe_embedding = time_elapsed(img_retrieval.compute_embedding, image)
compute_embeddings_times.append(elapsed)
elapsed, (sorted_indexes, distances) = time_elapsed(img_retrieval.search_in_gallery,
probe_embedding)
search_in_gallery_times.append(elapsed)
sorted_classes = [img_retrieval.gallery_classes[i] for i in sorted_indexes]
if args.ground_truth is not None:
position = sorted_classes.index(
img_retrieval.text_label_to_class_id[args.ground_truth])
positions.append(position)
log.info("ROI detected, found: %d, position of target: %d",
sorted_classes[0], position)
else:
log.info("ROI detected, found: %s", sorted_classes[0])
image, key = visualize(view_frame, position,
[img_retrieval.impaths[i] for i in sorted_indexes],
distances[sorted_indexes] if position is not None else None,
img_retrieval.input_size, np.mean(compute_embeddings_times),
np.mean(search_in_gallery_times), imshow_delay=3, presenter=presenter, no_show=args.no_show)
metrics.update(start_time)
if frames_processed == 0:
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), (image.shape[1], image.shape[0])):
raise RuntimeError("Can't open video writer")
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit):
video_writer.write(image)
if key == 27:
break
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
if positions:
compute_metrics(positions)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
next_frame_id = 1
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'remote':
log.info('Reading model {}'.format(args.model))
serving_config = {"address": "localhost", "port": 9000}
model_adapter = RemoteAdapter(args.model, serving_config)
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
model = Deblurring(model_adapter, preload=False)
model.reshape(frame.shape)
model.log_layers_info()
pipeline = AsyncPipeline(model)
pipeline.submit_data(frame, 0, {'frame': frame, 'start_time': start_time})
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(frame.shape[1] / 4), round(frame.shape[0] / 8)))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), (2 * frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
break
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {'frame': frame, 'start_time': start_time})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
result_frame, frame_meta = results
input_frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
if input_frame.shape != result_frame.shape:
input_frame = cv2.resize(input_frame, (result_frame.shape[1], result_frame.shape[0]))
final_image = cv2.hconcat([input_frame, result_frame])
render_metrics.update(rendering_start_time)
presenter.drawGraphs(final_image)
metrics.update(start_time, final_image)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(final_image)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Deblurring Results', final_image)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
result_frame, frame_meta = results
input_frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
if input_frame.shape != result_frame.shape:
input_frame = cv2.resize(input_frame, (result_frame.shape[1], result_frame.shape[0]))
final_image = cv2.hconcat([input_frame, result_frame])
render_metrics.update(rendering_start_time)
presenter.drawGraphs(final_image)
metrics.update(start_time, final_image)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(final_image)
if not args.no_show:
cv2.imshow('Deblurring Results', final_image)
key = cv2.waitKey(1)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
pipeline.preprocess_metrics.get_latency(),
pipeline.inference_metrics.get_latency(),
pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
# Plugin initialization for specified device and load extensions library if specified.
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, 'CPU')
# Read IR
log.info('Reading Mask-RCNN model {}'.format(args.mask_rcnn_model))
mask_rcnn_net = ie.read_network(args.mask_rcnn_model)
model_required_inputs = {'image'}
if set(mask_rcnn_net.input_info) == model_required_inputs:
required_output_keys = {'boxes', 'labels', 'masks', 'text_features.0'}
n, c, h, w = mask_rcnn_net.input_info['image'].input_data.shape
assert n == 1, 'Only batch 1 is supported by the demo application'
else:
raise RuntimeError(
'Demo supports only topologies with the following input keys: '
f'{model_required_inputs}.')
assert required_output_keys.issubset(mask_rcnn_net.outputs.keys()), \
f'Demo supports only topologies with the following output keys: {required_output_keys}' \
f'Found: {mask_rcnn_net.outputs.keys()}.'
log.info('Reading Text Recognition Encoder model {}'.format(
args.text_enc_model))
text_enc_net = ie.read_network(args.text_enc_model)
log.info('Reading Text Recognition Decoder model {}'.format(
args.text_dec_model))
text_dec_net = ie.read_network(args.text_dec_model)
mask_rcnn_exec_net = ie.load_network(network=mask_rcnn_net,
device_name=args.device,
num_requests=2)
log.info('The Mask-RCNN model {} is loaded to {}'.format(
args.mask_rcnn_model, args.device))
text_enc_exec_net = ie.load_network(network=text_enc_net,
device_name=args.device)
log.info('The Text Recognition Encoder model {} is loaded to {}'.format(
args.text_enc_model, args.device))
text_dec_exec_net = ie.load_network(network=text_dec_net,
device_name=args.device)
log.info('The Text Recognition Decoder model {} is loaded to {}'.format(
args.text_dec_model, args.device))
hidden_shape = text_dec_net.input_info[
args.trd_input_prev_hidden].input_data.shape
del mask_rcnn_net
del text_enc_net
del text_dec_net
if args.no_track:
tracker = None
else:
tracker = StaticIOUTracker()
if args.delay:
delay = args.delay
else:
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
visualizer = Visualizer(['__background__', 'text'],
show_boxes=args.show_boxes,
show_scores=args.show_scores)
frames_processed = 0
metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
presenter = monitors.Presenter(args.utilization_monitors, 45,
(frame.shape[1] // 4, frame.shape[0] // 8))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while frame is not None:
if not args.keep_aspect_ratio:
# Resize the image to a target size.
scale_x = w / frame.shape[1]
scale_y = h / frame.shape[0]
input_image = cv2.resize(frame, (w, h))
else:
# Resize the image to keep the same aspect ratio and to fit it to a window of a target size.
scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1])
input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y)
input_image_size = input_image.shape[:2]
input_image = np.pad(input_image,
((0, h - input_image_size[0]),
(0, w - input_image_size[1]), (0, 0)),
mode='constant',
constant_values=0)
# Change data layout from HWC to CHW.
input_image = input_image.transpose((2, 0, 1))
input_image = input_image.reshape((n, c, h, w)).astype(np.float32)
# Run the net.
outputs = mask_rcnn_exec_net.infer({'image': input_image})
# Parse detection results of the current request
boxes = outputs['boxes'][:, :4]
scores = outputs['boxes'][:, 4]
classes = outputs['labels'].astype(np.uint32)
raw_masks = outputs['masks']
text_features = outputs['text_features.0']
# Filter out detections with low confidence.
detections_filter = scores > args.prob_threshold
scores = scores[detections_filter]
classes = classes[detections_filter]
boxes = boxes[detections_filter]
raw_masks = raw_masks[detections_filter]
text_features = text_features[detections_filter]
boxes[:, 0::2] /= scale_x
boxes[:, 1::2] /= scale_y
masks = []
for box, cls, raw_mask in zip(boxes, classes, raw_masks):
mask = segm_postprocess(box, raw_mask, frame.shape[0],
frame.shape[1])
masks.append(mask)
texts = []
for feature in text_features:
feature = text_enc_exec_net.infer({'input': feature})['output']
feature = np.reshape(feature,
(feature.shape[0], feature.shape[1], -1))
feature = np.transpose(feature, (0, 2, 1))
hidden = np.zeros(hidden_shape)
prev_symbol_index = np.ones((1, )) * SOS_INDEX
text = ''
text_confidence = 1.0
for i in range(MAX_SEQ_LEN):
decoder_output = text_dec_exec_net.infer({
args.trd_input_prev_symbol:
prev_symbol_index,
args.trd_input_prev_hidden:
hidden,
args.trd_input_encoder_outputs:
feature
})
symbols_distr = decoder_output[args.trd_output_symbols_distr]
symbols_distr_softmaxed = softmax(symbols_distr, axis=1)[0]
prev_symbol_index = int(np.argmax(symbols_distr, axis=1))
text_confidence *= symbols_distr_softmaxed[prev_symbol_index]
if prev_symbol_index == EOS_INDEX:
break
text += args.alphabet[prev_symbol_index]
hidden = decoder_output[args.trd_output_cur_hidden]
texts.append(text if text_confidence >= args.tr_threshold else '')
if len(boxes) and args.raw_output_message:
log.debug(
' -------------------------- Frame # {} -------------------------- '
.format(frames_processed))
log.debug(
' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX '
)
for box, cls, score, mask in zip(boxes, classes, scores, masks):
log.debug(
'{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} '
.format(cls, score, *box))
# Get instance track IDs.
masks_tracks_ids = None
if tracker is not None:
masks_tracks_ids = tracker(masks, classes)
presenter.drawGraphs(frame)
# Visualize masks.
frame = visualizer(frame, boxes, classes, scores, masks, texts,
masks_tracks_ids)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
# Show resulting image.
cv2.imshow('Results', frame)
if not args.no_show:
key = cv2.waitKey(delay)
esc_code = 27
if key == esc_code:
break
presenter.handleKey(key)
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
cv2.destroyAllWindows()
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
delay = int(cap.get_type() in {'VIDEO', 'CAMERA'})
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'remote':
log.info('Reading model {}'.format(args.model))
serving_config = {"address": "localhost", "port": 9000}
model_adapter = RemoteAdapter(args.model, serving_config)
config = {
'mean_values': args.mean_values,
'scale_values': args.scale_values,
'reverse_input_channels': args.reverse_input_channels,
'topk': args.topk,
'path_to_labels': args.labels
}
model = Classification(model_adapter, config)
model.log_layers_info()
async_pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
ESC_KEY = 27
key = -1
while True:
if async_pipeline.callback_exceptions:
raise async_pipeline.callback_exceptions[0]
# Process all completed requests
results = async_pipeline.get_result(next_frame_id_to_show)
if results:
classifications, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if args.raw_output_message:
print_raw_results(classifications, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_labels(frame, classifications, output_transform)
if delay or args.no_show:
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Classification Results', frame)
key = cv2.waitKey(delay)
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
continue
if async_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
async_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
async_pipeline.await_any()
async_pipeline.await_all()
if key not in {ord('q'), ord('Q'), ESC_KEY}:
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show,
next_frame_id):
results = async_pipeline.get_result(next_frame_id_to_show)
while results is None:
results = async_pipeline.get_result(next_frame_id_to_show)
classifications, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if args.raw_output_message:
print_raw_results(classifications, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_labels(frame, classifications, output_transform)
if delay or args.no_show:
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Classification Results', frame)
key = cv2.waitKey(delay)
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
if delay or args.no_show:
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
async_pipeline.preprocess_metrics.get_latency(),
async_pipeline.inference_metrics.get_latency(),
async_pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def run_demo(args):
cap = open_images_capture(args.input, args.loop)
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
ie = IECore()
log.info('Reading Object Detection model {}'.format(args.model_od))
detector_person = Detector(ie,
args.model_od,
device=args.device,
label_class=args.person_label)
log.info('The Object Detection model {} is loaded to {}'.format(
args.model_od, args.device))
log.info('Reading Human Pose Estimation model {}'.format(args.model_hpe))
single_human_pose_estimator = HumanPoseEstimator(ie,
args.model_hpe,
device=args.device)
log.info('The Human Pose Estimation model {} is loaded to {}'.format(
args.model_hpe, args.device))
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
video_writer = cv2.VideoWriter()
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 25)
metrics = PerformanceMetrics()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while frame is not None:
bboxes = detector_person.detect(frame)
human_poses = [
single_human_pose_estimator.estimate(frame, bbox)
for bbox in bboxes
]
presenter.drawGraphs(frame)
colors = [(0, 0, 255), (255, 0, 0), (0, 255, 0), (255, 0, 0),
(0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0),
(0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0),
(0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0),
(0, 255, 0)]
for pose, bbox in zip(human_poses, bboxes):
cv2.rectangle(frame, (bbox[0], bbox[1]),
(bbox[0] + bbox[2], bbox[1] + bbox[3]), (255, 0, 0),
2)
for id_kpt, kpt in enumerate(pose):
cv2.circle(frame, (int(kpt[0]), int(kpt[1])), 3,
colors[id_kpt], -1)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Human Pose Estimation Demo', frame)
key = cv2.waitKey(delay)
if key == 27:
break
presenter.handleKey(key)
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'remote':
log.info('Reading model {}'.format(args.model))
serving_config = {"address": "localhost", "port": 9000}
model_adapter = RemoteAdapter(args.model, serving_config)
model = MonoDepthModel(model_adapter)
model.log_layers_info()
pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
pipeline.submit_data(frame, next_frame_id,
{'start_time': start_time})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
depth_map, frame_meta = results
depth_map = apply_color_map(depth_map, output_transform)
start_time = frame_meta['start_time']
presenter.drawGraphs(depth_map)
metrics.update(start_time, depth_map)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(depth_map)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow(DEMO_NAME, depth_map)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
depth_map, frame_meta = results
depth_map = apply_color_map(depth_map, output_transform)
start_time = frame_meta['start_time']
presenter.drawGraphs(depth_map)
metrics.update(start_time, depth_map)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(depth_map)
if not args.no_show:
cv2.imshow(DEMO_NAME, depth_map)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
frame_processor = FrameProcessor(args)
frame_num = 0
metrics = PerformanceMetrics()
presenter = None
output_transform = None
input_crop = None
if args.crop_size[0] > 0 and args.crop_size[1] > 0:
input_crop = np.array(args.crop_size)
elif not (args.crop_size[0] == 0 and args.crop_size[1] == 0):
raise ValueError('Both crop height and width should be positive')
video_writer = cv2.VideoWriter()
while True:
start_time = perf_counter()
frame = cap.read()
if frame is None:
if frame_num == 0:
raise ValueError("Can't read an image from the input")
break
if input_crop:
frame = center_crop(frame, input_crop)
if frame_num == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
output_resolution):
raise RuntimeError("Can't open video writer")
detections = frame_processor.process(frame)
presenter.drawGraphs(frame)
frame = draw_detections(frame, frame_processor, detections,
output_transform)
metrics.update(start_time, frame)
frame_num += 1
if video_writer.isOpened() and (args.output_limit <= 0
or frame_num <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Face recognition demo', frame)
key = cv2.waitKey(1)
# Quit
if key in {ord('q'), ord('Q'), 27}:
break
presenter.handleKey(key)
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
class_map = load_class_map(args.class_map)
assert class_map is not None
ie = load_ie_core(args.device, args.cpu_extension)
person_detector = PersonDetector(args.detection_model,
args.device,
ie,
num_requests=2,
output_shape=DETECTOR_OUTPUT_SHAPE)
action_recognizer = ActionRecognizer(args.action_model,
args.device,
ie,
num_requests=2,
img_scale=ACTION_IMAGE_SCALE,
num_classes=len(class_map))
person_tracker = Tracker(person_detector, TRACKER_SCORE_THRESHOLD,
TRACKER_IOU_THRESHOLD)
video_stream = VideoStream(args.input, ACTION_NET_INPUT_FPS,
action_recognizer.input_length)
video_stream.start()
metrics = PerformanceMetrics()
visualizer = Visualizer(VISUALIZER_TRG_FPS)
visualizer.register_window('Demo')
presenter = monitors.Presenter(args.utilization_monitors)
samples_library = None
if args.samples_dir is not None and os.path.exists(args.samples_dir):
visualizer.register_window('Gesture library')
visualizer.start()
library_queue = visualizer.get_queue('Gesture library')
samples_library = VideoLibrary(args.samples_dir,
SAMPLES_MAX_WINDOW_SIZE,
list(class_map.values()), library_queue,
SAMPLES_TRG_FPS)
samples_library.start()
else:
visualizer.start()
last_caption = None
active_object_id = -1
tracker_labels_map = {}
tracker_labels = set()
frames_processed = 0
while True:
start_time = perf_counter()
frame = video_stream.get_live_frame()
batch = video_stream.get_batch()
if frame is None or batch is None:
break
if frames_processed == 0:
video_writer = cv2.VideoWriter()
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
video_stream.fps(), (frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
detections, tracker_labels_map = person_tracker.add_frame(
frame, len(OBJECT_IDS), tracker_labels_map)
if detections is None:
active_object_id = -1
last_caption = None
if len(detections) == 1:
active_object_id = 0
if active_object_id >= 0:
cur_det = [det for det in detections if det.id == active_object_id]
if len(cur_det) != 1:
active_object_id = -1
last_caption = None
continue
recognizer_result = action_recognizer(batch,
cur_det[0].roi.reshape(-1))
if recognizer_result is not None:
action_class_id = np.argmax(recognizer_result)
action_class_label = \
class_map[action_class_id] if class_map is not None else action_class_id
action_class_score = np.max(recognizer_result)
if action_class_score > args.action_threshold:
last_caption = 'Last gesture: {} '.format(
action_class_label)
presenter.drawGraphs(frame)
if detections is not None:
tracker_labels = {det.id for det in detections}
for det in detections:
roi_color = (0, 255,
0) if active_object_id == det.id else (128, 128,
128)
border_width = 2 if active_object_id == det.id else 1
person_roi = det.roi[0]
cv2.rectangle(frame, (person_roi[0], person_roi[1]),
(person_roi[2], person_roi[3]), roi_color,
border_width)
cv2.putText(frame, str(det.id),
(person_roi[0] + 10, person_roi[1] + 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.8, roi_color, 2)
if last_caption is not None:
cv2.putText(frame, last_caption, (10, frame.shape[0] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if args.no_show:
continue
visualizer.put_queue(frame, 'Demo')
key = visualizer.get_key()
if key == 27: # esc
break
elif key == ord(' '): # space
active_object_id = -1
last_caption = None
elif key == 13: # enter
last_caption = None
elif key in OBJECT_IDS: # 0-9
local_bbox_id = int(chr(key))
if local_bbox_id in tracker_labels:
active_object_id = local_bbox_id
else:
presenter.handleKey(key)
if samples_library is not None:
if key == ord('f'): # forward
samples_library.next()
elif key == ord('b'): # backward
samples_library.prev()
if samples_library is not None:
samples_library.release()
visualizer.release()
video_stream.release()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
with open(args.labels, 'rt') as labels_file:
class_labels = labels_file.read().splitlines()
assert len(class_labels), 'The file with class labels is empty'
# Plugin initialization for specified device and load extensions library if specified.
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, 'CPU')
# Read IR
log.info('Reading model {}'.format(args.model))
net = ie.read_network(args.model, args.model.with_suffix('.bin'))
image_input, image_info_input, (
n, c, h, w), model_type, postprocessor = check_model(net)
args.no_keep_aspect_ratio = model_type == 'yolact' or args.no_keep_aspect_ratio
exec_net = ie.load_network(network=net,
device_name=args.device,
num_requests=2)
log.info('The model {} is loaded to {}'.format(args.model, args.device))
if args.no_track:
tracker = None
else:
tracker = StaticIOUTracker()
if args.delay:
delay = args.delay
else:
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
frames_processed = 0
metrics = PerformanceMetrics()
visualizer = Visualizer(class_labels,
show_boxes=args.show_boxes,
show_scores=args.show_scores)
video_writer = cv2.VideoWriter()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
out_frame_size = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 45,
(round(out_frame_size[0] / 4), round(out_frame_size[1] / 8)))
if args.output and not video_writer.open(args.output,
cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), out_frame_size):
raise RuntimeError("Can't open video writer")
while frame is not None:
if args.no_keep_aspect_ratio:
# Resize the image to a target size.
scale_x = w / frame.shape[1]
scale_y = h / frame.shape[0]
input_image = cv2.resize(frame, (w, h))
else:
# Resize the image to keep the same aspect ratio and to fit it to a window of a target size.
scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1])
input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y)
input_image_size = input_image.shape[:2]
input_image = np.pad(input_image,
((0, h - input_image_size[0]),
(0, w - input_image_size[1]), (0, 0)),
mode='constant',
constant_values=0)
# Change data layout from HWC to CHW.
input_image = input_image.transpose((2, 0, 1))
input_image = input_image.reshape((n, c, h, w)).astype(np.float32)
input_image_info = np.asarray(
[[input_image_size[0], input_image_size[1], 1]], dtype=np.float32)
# Run the net.
feed_dict = {image_input: input_image}
if image_info_input:
feed_dict[image_info_input] = input_image_info
outputs = exec_net.infer(feed_dict)
# Parse detection results of the current request
scores, classes, boxes, masks = postprocessor(outputs, scale_x,
scale_y,
*frame.shape[:2], h, w,
args.prob_threshold)
if len(boxes) and args.raw_output_message:
log.debug(
' -------------------------- Frame # {} -------------------------- '
.format(frames_processed))
log.debug(
' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX '
)
for box, cls, score, mask in zip(boxes, classes, scores, masks):
log.debug(
'{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} '
.format(cls, score, *box))
# Get instance track IDs.
masks_tracks_ids = None
if tracker is not None:
masks_tracks_ids = tracker(masks, classes)
# Visualize masks.
frame = visualizer(frame, boxes, classes, scores, presenter, masks,
masks_tracks_ids)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
# Show resulting image.
cv2.imshow('Results', frame)
if not args.no_show:
key = cv2.waitKey(delay)
esc_code = 27
if key == esc_code:
break
presenter.handleKey(key)
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
cv2.destroyAllWindows()
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
place_recognition = PlaceRecognition(args.model, args.device,
args.gallery_folder,
args.cpu_extension, args.gallery_size)
compute_embeddings_times = []
search_in_gallery_times = []
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 0)
video_writer = cv2.VideoWriter()
metrics = PerformanceMetrics()
while True:
start_time = perf_counter()
frame = cap.read()
if frame is None:
if frames_processed == 0:
raise ValueError("Can't read an image from the input")
break
elapsed, probe_embedding = time_elapsed(
place_recognition.compute_embedding, frame)
compute_embeddings_times.append(elapsed)
elapsed, (sorted_indexes, distances) = time_elapsed(
place_recognition.search_in_gallery, probe_embedding)
search_in_gallery_times.append(elapsed)
image, key = visualize(
frame, [str(place_recognition.impaths[i]) for i in sorted_indexes],
distances[sorted_indexes],
place_recognition.input_size,
np.mean(compute_embeddings_times),
np.mean(search_in_gallery_times),
imshow_delay=3,
presenter=presenter,
no_show=args.no_show)
metrics.update(start_time)
if frames_processed == 0:
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(image.shape[1], image.shape[0])):
raise RuntimeError("Can't open video writer")
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(image)
if key == 27:
break
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def run(params, config, capture, detector, reid):
win_name = 'Multi camera tracking'
frame_number = 0
output_detections = [[] for _ in range(capture.get_num_sources())]
key = -1
if config.normalizer_config.enabled:
capture.add_transform(
NormalizerCLAHE(
config.normalizer_config.clip_limit,
config.normalizer_config.tile_size,
))
tracker = MultiCameraTracker(capture.get_num_sources(),
reid,
config.sct_config,
**vars(config.mct_config),
visual_analyze=config.analyzer)
thread_body = FramesThreadBody(capture,
max_queue_length=len(capture.captures) * 2)
frames_thread = Thread(target=thread_body)
frames_thread.start()
frames_read = False
set_output_params = False
prev_frames = thread_body.frames_queue.get()
detector.run_async(prev_frames, frame_number)
metrics = PerformanceMetrics()
presenter = monitors.Presenter(params.utilization_monitors, 0)
while thread_body.process:
if not params.no_show:
key = check_pressed_keys(key)
if key == 27:
break
presenter.handleKey(key)
start_time = time.perf_counter()
try:
frames = thread_body.frames_queue.get_nowait()
frames_read = True
except queue.Empty:
frames = None
if frames is None:
continue
all_detections = detector.wait_and_grab()
if params.save_detections:
update_detections(output_detections, all_detections, frame_number)
frame_number += 1
detector.run_async(frames, frame_number)
all_masks = [[] for _ in range(len(all_detections))]
for i, detections in enumerate(all_detections):
all_detections[i] = [det[0] for det in detections]
all_masks[i] = [det[2] for det in detections if len(det) == 3]
tracker.process(prev_frames, all_detections, all_masks)
tracked_objects = tracker.get_tracked_objects()
vis = visualize_multicam_detections(
prev_frames, tracked_objects, **vars(config.visualization_config))
metrics.update(start_time, vis)
presenter.drawGraphs(vis)
if not params.no_show:
cv.imshow(win_name, vis)
if frames_read and not set_output_params:
set_output_params = True
if len(params.output_video):
frame_size = [frame.shape[::-1] for frame in frames]
fps = capture.get_fps()
target_width, target_height = get_target_size(
frame_size, None, **vars(config.visualization_config))
video_output_size = (target_width, target_height)
fourcc = cv.VideoWriter_fourcc(*'XVID')
output_video = cv.VideoWriter(params.output_video, fourcc,
min(fps), video_output_size)
else:
output_video = None
if set_output_params and output_video:
output_video.write(cv.resize(vis, video_output_size))
prev_frames, frames = frames, prev_frames
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
thread_body.process = False
frames_thread.join()
if len(params.history_file):
save_json_file(params.history_file,
tracker.get_all_tracks_history(),
description='History file')
if len(params.save_detections):
save_json_file(params.save_detections,
output_detections,
description='Detections')
if len(config.embeddings.save_path):
save_embeddings(tracker.scts, **vars(config.embeddings))
assert len(load_net.outputs) == 1, "Expected number of outputs is equal 1"
output_blob = next(iter(load_net.outputs))
output_shape = load_net.outputs[output_blob].shape
exec_net = ie.load_network(network=load_net, device_name=args.device)
log.info('The model {} is loaded to {}'.format(args.model, args.device))
_, _, h_in, w_in = input_shape
frames_processed = 0
imshow_size = (640, 480)
graph_size = (imshow_size[0] // 2, imshow_size[1] // 4)
presenter = monitors.Presenter(args.utilization_monitors,
imshow_size[1] * 2 - graph_size[1],
graph_size)
metrics = PerformanceMetrics()
video_writer = cv.VideoWriter()
if args.output and not video_writer.open(
args.output, cv.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(imshow_size[0] * 2, imshow_size[1] * 2)):
raise RuntimeError("Can't open video writer")
start_time = perf_counter()
original_frame = cap.read()
if original_frame is None:
raise RuntimeError("Can't read an image from the input")
while original_frame is not None:
(h_orig, w_orig) = original_frame.shape[:2]
R = np.array(extrinsics['R'], dtype=np.float32)
t = np.array(extrinsics['t'], dtype=np.float32)
is_video = cap.get_type() in ('VIDEO', 'CAMERA')
base_height = args.height_size
fx = args.fx
frames_processed = 0
delay = 1
esc_code = 27
p_code = 112
space_code = 32
mean_time = 0
presenter = monitors.Presenter(args.utilization_monitors, 0)
metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while frame is not None:
current_time = cv2.getTickCount()
input_scale = base_height / frame.shape[0]
def main():
parser = argparse.ArgumentParser(description='Whiteboard inpainting demo')
parser.add_argument(
'-i',
'--input',
required=True,
help='Required. Path to a video file or a device node of a web-camera.'
)
parser.add_argument('--loop',
default=False,
action='store_true',
help='Optional. Enable reading the input in a loop.')
parser.add_argument('-o',
'--output',
required=False,
help='Optional. Name of the output file(s) to save.')
parser.add_argument('-limit',
'--output_limit',
required=False,
default=1000,
type=int,
help='Optional. Number of frames to store in output. '
'If 0 is set, all frames are stored.')
parser.add_argument(
'-m_i',
'--m_instance_segmentation',
type=str,
required=False,
help='Required. Path to the instance segmentation model.')
parser.add_argument(
'-m_s',
'--m_semantic_segmentation',
type=str,
required=False,
help='Required. Path to the semantic segmentation model.')
parser.add_argument(
'-t',
'--threshold',
type=float,
default=0.6,
help='Optional. Threshold for person instance segmentation model.')
parser.add_argument('--no_show',
help="Optional. Don't show output.",
action='store_true')
parser.add_argument(
'-d',
'--device',
type=str,
default='CPU',
help=
'Optional. Specify a target device to infer on. CPU, GPU, HDDL or MYRIAD is '
'acceptable. The demo will look for a suitable plugin for the device specified.'
)
parser.add_argument('-l',
'--cpu_extension',
type=str,
default=None,
help='MKLDNN (CPU)-targeted custom layers. Absolute \
path to a shared library with the kernels impl.')
parser.add_argument('-u',
'--utilization_monitors',
default='',
type=str,
help='Optional. List of monitors to show initially.')
args = parser.parse_args()
cap = open_images_capture(args.input, args.loop)
if cap.get_type() not in ('VIDEO', 'CAMERA'):
raise RuntimeError(
"The input should be a video file or a numeric camera ID")
if bool(args.m_instance_segmentation) == bool(
args.m_semantic_segmentation):
raise ValueError(
'Set up exactly one of segmentation models: '
'--m_instance_segmentation or --m_semantic_segmentation')
labels_dir = Path(__file__).resolve().parents[3] / 'data/dataset_classes'
mouse = MouseClick()
if not args.no_show:
cv2.namedWindow(WINNAME)
cv2.setMouseCallback(WINNAME, mouse.get_points)
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
ie = IECore()
model_path = args.m_instance_segmentation if args.m_instance_segmentation else args.m_semantic_segmentation
log.info('Reading model {}'.format(model_path))
if args.m_instance_segmentation:
labels_file = str(labels_dir / 'coco_80cl_bkgr.txt')
segmentation = MaskRCNN(ie, args.m_instance_segmentation, labels_file,
args.threshold, args.device,
args.cpu_extension)
elif args.m_semantic_segmentation:
labels_file = str(labels_dir / 'cityscapes_19cl_bkgr.txt')
segmentation = SemanticSegmentation(ie, args.m_semantic_segmentation,
labels_file, args.threshold,
args.device, args.cpu_extension)
log.info('The model {} is loaded to {}'.format(model_path, args.device))
metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
black_board = False
frame_number = 0
key = -1
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
out_frame_size = (frame.shape[1], frame.shape[0] * 2)
output_frame = np.full((frame.shape[0], frame.shape[1], 3),
255,
dtype='uint8')
presenter = monitors.Presenter(
args.utilization_monitors, 20,
(out_frame_size[0] // 4, out_frame_size[1] // 16))
if args.output and not video_writer.open(args.output,
cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), out_frame_size):
raise RuntimeError("Can't open video writer")
while frame is not None:
mask = None
detections = segmentation.get_detections([frame])
expand_mask(detections, frame.shape[1] // 27)
if len(detections[0]) > 0:
mask = detections[0][0][2]
for i in range(1, len(detections[0])):
mask = cv2.bitwise_or(mask, detections[0][i][2])
if mask is not None:
mask = np.stack([mask, mask, mask], axis=-1)
else:
mask = np.zeros(frame.shape, dtype='uint8')
clear_frame = remove_background(frame, invert_colors=not black_board)
output_frame = np.where(mask, output_frame, clear_frame)
merged_frame = np.vstack([frame, output_frame])
merged_frame = cv2.resize(merged_frame, out_frame_size)
metrics.update(start_time, merged_frame)
if video_writer.isOpened() and (args.output_limit <= 0 or
frame_number <= args.output_limit - 1):
video_writer.write(merged_frame)
presenter.drawGraphs(merged_frame)
if not args.no_show:
cv2.imshow(WINNAME, merged_frame)
key = check_pressed_keys(key)
if key == 27: # 'Esc'
break
if key == ord('i'): # catch pressing of key 'i'
black_board = not black_board
output_frame = 255 - output_frame
else:
presenter.handleKey(key)
if mouse.crop_available:
x0, x1 = min(mouse.points[0][0], mouse.points[1][0]), \
max(mouse.points[0][0], mouse.points[1][0])
y0, y1 = min(mouse.points[0][1], mouse.points[1][1]), \
max(mouse.points[0][1], mouse.points[1][1])
x1, y1 = min(x1, output_frame.shape[1] - 1), min(
y1, output_frame.shape[0] - 1)
board = output_frame[y0:y1, x0:x1, :]
if board.shape[0] > 0 and board.shape[1] > 0:
cv2.namedWindow('Board', cv2.WINDOW_KEEPRATIO)
cv2.imshow('Board', board)
frame_number += 1
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)