def __init__(self, sources, loop):
assert sources
self.captures = []
self.transforms = []
self.fps = []
for src in sources:
capture = open_images_capture(src, loop)
self.captures.append(capture)
self.fps.append(capture.fps())
def main():
args = build_argparser().parse_args()
if args.labels:
with open(args.labels) as f:
labels = [line.strip() for line in f]
else:
labels = None
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
if 'MYRIAD' in args.device:
myriad_config = {'VPU_HW_STAGES_OPTIMIZATION': 'YES'}
core.set_config(myriad_config, 'MYRIAD')
decoder_target_device = 'CPU'
if args.device != 'CPU':
encoder_target_device = args.device
else:
encoder_target_device = decoder_target_device
models = [IEModel(args.m_encoder, core, encoder_target_device, model_type='Action Recognition Encoder',
num_requests=(3 if args.device == 'MYRIAD' else 1))]
if args.architecture_type == 'en-de':
if args.m_decoder is None:
raise RuntimeError('No decoder for encoder-decoder model type (-m_de) provided')
models.append(IEModel(args.m_decoder, core, decoder_target_device, model_type='Action Recognition Decoder', num_requests=2))
seq_size = models[1].input_shape[1]
elif args.architecture_type == 'en-mean':
models.append(DummyDecoder(num_requests=2))
seq_size = args.decoder_seq_size
elif args.architecture_type == 'i3d-rgb':
seq_size = models[0].input_shape[1]
presenter = monitors.Presenter(args.utilization_monitors, 70)
result_presenter = ResultRenderer(no_show=args.no_show, presenter=presenter, output=args.output, limit=args.output_limit, labels=labels,
label_smoothing_window=args.label_smoothing)
cap = open_images_capture(args.input, args.loop)
run_pipeline(cap, args.architecture_type, models, result_presenter.render_frame, args.raw_output_message,
seq_size=seq_size, fps=cap.fps())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
log.info('Initializing Inference Engine...')
ie = IECore()
plugin_config = get_plugin_configs(args.device, args.num_streams,
args.num_threads)
log.info('Loading network...')
model = get_model(ie, args)
detector_pipeline = AsyncPipeline(ie,
model,
plugin_config,
device=args.device,
max_num_requests=args.num_infer_requests)
### READ TIME ###
read_time_start = time.time()
cap = open_images_capture(args.input, args.loop)
read_time_end = time.time()
next_frame_id = 0
next_frame_id_to_show = 0
image_id = 0
log.info('Starting inference...')
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
palette = ColorPalette(len(model.labels) if model.labels else 100)
metrics = PerformanceMetrics()
presenter = None
video_writer = cv2.VideoWriter()
results_list = []
detection_ids = [1, 3, 4]
all_starts = 0
while True:
print('NEXT FRAME ID', next_frame_id)
id = images[image_id]
if next_frame_id == 5000:
break
if detector_pipeline.callback_exceptions:
raise detector_pipeline.callback_exceptions[0]
# Process all completed requests
#### DETECTION TIME ####
detect_time_start = time.time()
results = detector_pipeline.get_result(next_frame_id_to_show)
detect_time_end = time.time()
detect_time_list.append(detect_time_end - detect_time_start)
if results:
objects, frame_meta = results
for detection in objects:
x = float(detection.xmin)
y = float(detection.ymin)
w = float(detection.xmax - detection.xmin)
h = float(detection.ymax - detection.ymin)
cls = detection.id
cls = yolo_to_ssd_classes[cls]
id = str(id.lstrip('0').split('.')[0])
conf = detection.score
# if cls in detection_ids:
results_list.append({
'image_id': int(id),
'category_id': cls,
'bbox': [x, y, w, h],
'score': float(conf)
})
frame = frame_meta['frame']
post_process_start = time.time()
start_time = frame_meta['start_time']
all_starts += start_time
all_starts += start_time
if len(objects) and args.raw_output_message:
print_raw_results(frame.shape[:2], objects, model.labels,
args.prob_threshold, images[image_id])
presenter.drawGraphs(frame)
frame = draw_detections(frame, objects, palette, model.labels,
args.prob_threshold, images[image_id])
metrics.update(start_time, frame)
post_process_end = time.time()
post_process_list.append(post_process_end - post_process_start)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
# cv2.imshow('Detection Results', frame)
cv2.imwrite(
f"/home/sovit/my_data/Data_Science/Projects/openvino_experiments/model_quantization/data/images/image_{image_id}.jpg",
frame)
# key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
#if key in {ord('q'), ord('Q'), ESC_KEY}:
#break
#presenter.handleKey(key)
next_frame_id_to_show += 1
image_id += 1
continue
if detector_pipeline.is_ready():
# Get new image/frame
pre_process_start = time.time()
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(frame.shape[1] / 4), round(frame.shape[0] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), (frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
# Submit for inference
detector_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
pre_process_end = time.time()
pre_process_list.append(pre_process_end - pre_process_start)
next_frame_id += 1
else:
# Wait for empty request
detector_pipeline.await_any()
results_file = 'results.json'
with open(results_file, 'w') as f:
f.write(json.dumps(results_list, indent=4))
detector_pipeline.await_all()
# Process completed requests
while detector_pipeline.has_completed_request():
results = detector_pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
frame = frame_meta['frame']
post_process_two_start = time.time()
start_time = frame_meta['start_time']
if len(objects) and args.raw_output_message:
print()
# print_raw_results(frame.shape[:2], objects, model.labels, args.prob_threshold)
presenter.drawGraphs(frame)
# frame = draw_detections(frame, objects, palette, model.labels, args.prob_threshold)
metrics.update(start_time, frame)
post_process_two_end = time.time()
post_process_list_two.append(post_process_two_end -
post_process_two_start)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
# cv2.imshow('Detection Results', frame)
cv2.imwrite(
f"/home/sovit/my_data/Data_Science/Projects/openvino_experiments/model_quantization/data/images/image_{frame_id}.jpg",
frame)
# key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
next_frame_id_to_show += 1
else:
break
metrics.print_total()
print("Presentor", presenter.reportMeans())
def main():
""" Main function. """
log.basicConfig(format='[ %(levelname)s ] %(message)s',
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
place_recognition = PlaceRecognition(args.model, args.device,
args.gallery_folder,
args.cpu_extension, args.gallery_size)
cap = open_images_capture(args.input, args.loop)
compute_embeddings_times = []
search_in_gallery_times = []
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 0)
video_writer = cv2.VideoWriter()
while True:
frame = cap.read()
if frame is None:
if frames_processed == 0:
raise ValueError("Can't read an image from the input")
break
elapsed, probe_embedding = time_elapsed(
place_recognition.compute_embedding, frame)
compute_embeddings_times.append(elapsed)
elapsed, (sorted_indexes, distances) = time_elapsed(
place_recognition.search_in_gallery, probe_embedding)
search_in_gallery_times.append(elapsed)
image, key = visualize(
frame, [str(place_recognition.impaths[i]) for i in sorted_indexes],
distances[sorted_indexes],
place_recognition.input_size,
np.mean(compute_embeddings_times),
np.mean(search_in_gallery_times),
imshow_delay=3,
presenter=presenter,
no_show=args.no_show)
if frames_processed == 0:
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(image.shape[1], image.shape[0])):
raise RuntimeError("Can't open video writer")
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(image)
if key == 27:
break
print(presenter.reportMeans())
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
next_frame_id = 1
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests,
model_parameters={'input_layouts': args.layout})
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
config = {
'target_size': args.tsize,
'aspect_ratio': frame.shape[1] / frame.shape[0],
'confidence_threshold': args.prob_threshold,
'padding_mode': 'center' if args.architecture_type == 'higherhrnet'
else None, # the 'higherhrnet' and 'ae' specific
'delta': 0.5 if args.architecture_type == 'higherhrnet' else
None, # the 'higherhrnet' and 'ae' specific
}
model = ImageModel.create_model(ARCHITECTURES[args.architecture_type],
model_adapter, config)
model.log_layers_info()
hpe_pipeline = AsyncPipeline(model)
hpe_pipeline.submit_data(frame, 0, {
'frame': frame,
'start_time': start_time
})
output_transform = OutputTransform(frame.shape[:2], args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
if args.output and not video_writer.open(args.output,
cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
while True:
if hpe_pipeline.callback_exceptions:
raise hpe_pipeline.callback_exceptions[0]
# Process all completed requests
results = hpe_pipeline.get_result(next_frame_id_to_show)
if results:
(poses, scores), frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(poses) and args.raw_output_message:
print_raw_results(poses, scores, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_poses(frame, poses, args.prob_threshold,
output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Pose estimation results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
continue
if hpe_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
break
# Submit for inference
hpe_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
hpe_pipeline.await_any()
hpe_pipeline.await_all()
if hpe_pipeline.callback_exceptions:
raise hpe_pipeline.callback_exceptions[0]
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = hpe_pipeline.get_result(next_frame_id_to_show)
(poses, scores), frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(poses) and args.raw_output_message:
print_raw_results(poses, scores, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_poses(frame, poses, args.prob_threshold, output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Pose estimation results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
hpe_pipeline.preprocess_metrics.get_latency(),
hpe_pipeline.inference_metrics.get_latency(),
hpe_pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def run_demo(args):
ie = IECore()
detector_person = Detector(ie,
path_to_model_xml=args.model_od,
device=args.device,
label_class=args.person_label)
single_human_pose_estimator = HumanPoseEstimator(
ie, path_to_model_xml=args.model_hpe, device=args.device)
cap = open_images_capture(args.input, args.loop)
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
video_writer = cv2.VideoWriter()
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 25)
while frame is not None:
bboxes = detector_person.detect(frame)
human_poses = [
single_human_pose_estimator.estimate(frame, bbox)
for bbox in bboxes
]
presenter.drawGraphs(frame)
colors = [(0, 0, 255), (255, 0, 0), (0, 255, 0), (255, 0, 0),
(0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0),
(0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0),
(0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0),
(0, 255, 0)]
for pose, bbox in zip(human_poses, bboxes):
cv2.rectangle(frame, (bbox[0], bbox[1]),
(bbox[0] + bbox[2], bbox[1] + bbox[3]), (255, 0, 0),
2)
for id_kpt, kpt in enumerate(pose):
cv2.circle(frame, (int(kpt[0]), int(kpt[1])), 3,
colors[id_kpt], -1)
cv2.putText(
frame,
'summary: {:.1f} FPS (estimation: {:.1f} FPS / detection: {:.1f} FPS)'
.format(
float(1 / (detector_person.infer_time +
single_human_pose_estimator.infer_time *
len(human_poses))),
float(1 / single_human_pose_estimator.infer_time),
float(1 / detector_person.infer_time)), (5, 15),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 200))
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Human Pose Estimation Demo', frame)
key = cv2.waitKey(delay)
if key == 27:
break
presenter.handleKey(key)
frame = cap.read()
print(presenter.reportMeans())
def main():
log.basicConfig(format='[ %(levelname)s ] %(message)s',
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
# Plugin initialization for specified device and load extensions library if specified.
log.info('Creating Inference Engine...')
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, 'CPU')
# Read IR
log.info('Loading Mask-RCNN network')
mask_rcnn_net = ie.read_network(
args.mask_rcnn_model,
os.path.splitext(args.mask_rcnn_model)[0] + '.bin')
log.info('Loading encoder part of text recognition network')
text_enc_net = ie.read_network(
args.text_enc_model,
os.path.splitext(args.text_enc_model)[0] + '.bin')
log.info('Loading decoder part of text recognition network')
text_dec_net = ie.read_network(
args.text_dec_model,
os.path.splitext(args.text_dec_model)[0] + '.bin')
model_required_inputs = {'image'}
if set(mask_rcnn_net.input_info) == model_required_inputs:
required_output_keys = {'boxes', 'labels', 'masks', 'text_features.0'}
n, c, h, w = mask_rcnn_net.input_info['image'].input_data.shape
else:
raise RuntimeError(
'Demo supports only topologies with the following input keys: '
f'{model_required_inputs}.')
assert required_output_keys.issubset(mask_rcnn_net.outputs.keys()), \
f'Demo supports only topologies with the following output keys: {required_output_keys}' \
f'Found: {mask_rcnn_net.outputs.keys()}.'
assert n == 1, 'Only batch 1 is supported by the demo application'
log.info('Loading IR to the plugin...')
mask_rcnn_exec_net = ie.load_network(network=mask_rcnn_net,
device_name=args.device,
num_requests=2)
text_enc_exec_net = ie.load_network(network=text_enc_net,
device_name=args.device)
text_dec_exec_net = ie.load_network(network=text_dec_net,
device_name=args.device)
hidden_shape = text_dec_net.input_info[
args.trd_input_prev_hidden].input_data.shape
del mask_rcnn_net
del text_enc_net
del text_dec_net
cap = open_images_capture(args.input, args.loop)
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
if args.no_track:
tracker = None
else:
tracker = StaticIOUTracker()
if args.delay:
delay = args.delay
else:
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
visualizer = Visualizer(['__background__', 'text'],
show_boxes=args.show_boxes,
show_scores=args.show_scores)
render_time = 0
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 45,
(frame.shape[1] // 4, frame.shape[0] // 8))
video_writer = cv2.VideoWriter()
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
log.info('Starting inference...')
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
while frame is not None:
if not args.keep_aspect_ratio:
# Resize the image to a target size.
scale_x = w / frame.shape[1]
scale_y = h / frame.shape[0]
input_image = cv2.resize(frame, (w, h))
else:
# Resize the image to keep the same aspect ratio and to fit it to a window of a target size.
scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1])
input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y)
input_image_size = input_image.shape[:2]
input_image = np.pad(input_image,
((0, h - input_image_size[0]),
(0, w - input_image_size[1]), (0, 0)),
mode='constant',
constant_values=0)
# Change data layout from HWC to CHW.
input_image = input_image.transpose((2, 0, 1))
input_image = input_image.reshape((n, c, h, w)).astype(np.float32)
# Run the net.
inf_start = time.time()
outputs = mask_rcnn_exec_net.infer({'image': input_image})
# Parse detection results of the current request
boxes = outputs['boxes'][:, :4]
scores = outputs['boxes'][:, 4]
classes = outputs['labels'].astype(np.uint32)
raw_masks = outputs['masks']
text_features = outputs['text_features.0']
# Filter out detections with low confidence.
detections_filter = scores > args.prob_threshold
scores = scores[detections_filter]
classes = classes[detections_filter]
boxes = boxes[detections_filter]
raw_masks = raw_masks[detections_filter]
text_features = text_features[detections_filter]
boxes[:, 0::2] /= scale_x
boxes[:, 1::2] /= scale_y
masks = []
for box, cls, raw_mask in zip(boxes, classes, raw_masks):
mask = segm_postprocess(box, raw_mask, frame.shape[0],
frame.shape[1])
masks.append(mask)
texts = []
for feature in text_features:
feature = text_enc_exec_net.infer({'input': feature})['output']
feature = np.reshape(feature,
(feature.shape[0], feature.shape[1], -1))
feature = np.transpose(feature, (0, 2, 1))
hidden = np.zeros(hidden_shape)
prev_symbol_index = np.ones((1, )) * SOS_INDEX
text = ''
text_confidence = 1.0
for i in range(MAX_SEQ_LEN):
decoder_output = text_dec_exec_net.infer({
args.trd_input_prev_symbol:
prev_symbol_index,
args.trd_input_prev_hidden:
hidden,
args.trd_input_encoder_outputs:
feature
})
symbols_distr = decoder_output[args.trd_output_symbols_distr]
symbols_distr_softmaxed = softmax(symbols_distr, axis=1)[0]
prev_symbol_index = int(np.argmax(symbols_distr, axis=1))
text_confidence *= symbols_distr_softmaxed[prev_symbol_index]
if prev_symbol_index == EOS_INDEX:
break
text += args.alphabet[prev_symbol_index]
hidden = decoder_output[args.trd_output_cur_hidden]
texts.append(text if text_confidence >= args.tr_threshold else '')
inf_end = time.time()
inf_time = inf_end - inf_start
render_start = time.time()
if len(boxes) and args.raw_output_message:
log.info('Detected boxes:')
log.info(
' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX '
)
for box, cls, score, mask in zip(boxes, classes, scores, masks):
log.info(
'{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} '
.format(cls, score, *box))
# Get instance track IDs.
masks_tracks_ids = None
if tracker is not None:
masks_tracks_ids = tracker(masks, classes)
presenter.drawGraphs(frame)
# Visualize masks.
frame = visualizer(frame, boxes, classes, scores, masks, texts,
masks_tracks_ids)
# Draw performance stats.
inf_time_message = 'Inference and post-processing time: {:.3f} ms'.format(
inf_time * 1000)
render_time_message = 'OpenCV rendering time: {:.3f} ms'.format(
render_time * 1000)
cv2.putText(frame, inf_time_message, (15, 15),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1)
cv2.putText(frame, render_time_message, (15, 30),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
# Print performance counters.
if args.perf_counts:
perf_counts = mask_rcnn_exec_net.requests[0].get_perf_counts()
log.info('Performance counters:')
print('{:<70} {:<15} {:<15} {:<15} {:<10}'.format(
'name', 'layer_type', 'exet_type', 'status', 'real_time, us'))
for layer, stats in perf_counts.items():
print('{:<70} {:<15} {:<15} {:<15} {:<10}'.format(
layer, stats['layer_type'], stats['exec_type'],
stats['status'], stats['real_time']))
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
# Show resulting image.
cv2.imshow('Results', frame)
render_end = time.time()
render_time = render_end - render_start
if not args.no_show:
key = cv2.waitKey(delay)
esc_code = 27
if key == esc_code:
break
presenter.handleKey(key)
frame = cap.read()
print(presenter.reportMeans())
cv2.destroyAllWindows()
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
next_frame_id = 1
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
model = Deblurring(model_adapter, preload=False)
model.reshape(frame.shape)
model.log_layers_info()
pipeline = AsyncPipeline(model)
pipeline.submit_data(frame, 0, {'frame': frame, 'start_time': start_time})
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(frame.shape[1] / 4), round(frame.shape[0] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(2 * frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
break
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
result_frame, frame_meta = results
input_frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
if input_frame.shape != result_frame.shape:
input_frame = cv2.resize(
input_frame,
(result_frame.shape[1], result_frame.shape[0]))
final_image = cv2.hconcat([input_frame, result_frame])
render_metrics.update(rendering_start_time)
presenter.drawGraphs(final_image)
metrics.update(start_time, final_image)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(final_image)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Deblurring Results', final_image)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
result_frame, frame_meta = results
input_frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
if input_frame.shape != result_frame.shape:
input_frame = cv2.resize(
input_frame, (result_frame.shape[1], result_frame.shape[0]))
final_image = cv2.hconcat([input_frame, result_frame])
render_metrics.update(rendering_start_time)
presenter.drawGraphs(final_image)
metrics.update(start_time, final_image)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(final_image)
if not args.no_show:
cv2.imshow('Deblurring Results', final_image)
key = cv2.waitKey(1)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
pipeline.preprocess_metrics.get_latency(),
pipeline.inference_metrics.get_latency(),
pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def run_demo(args):
cap = open_images_capture(args.input, args.loop)
log.info('OpenVINO Runtime')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
log.info('Reading Object Detection model {}'.format(args.model_od))
detector_person = Detector(core, args.model_od,
device=args.device,
label_class=args.person_label)
log.info('The Object Detection model {} is loaded to {}'.format(args.model_od, args.device))
log.info('Reading Human Pose Estimation model {}'.format(args.model_hpe))
single_human_pose_estimator = HumanPoseEstimator(core, args.model_hpe,
device=args.device)
log.info('The Human Pose Estimation model {} is loaded to {}'.format(args.model_hpe, args.device))
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
video_writer = cv2.VideoWriter()
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 25)
metrics = PerformanceMetrics()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), (frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while frame is not None:
bboxes = detector_person.detect(frame)
human_poses = [single_human_pose_estimator.estimate(frame, bbox) for bbox in bboxes]
presenter.drawGraphs(frame)
colors = [(0, 0, 255),
(255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0),
(255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0),
(255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0),
(255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0)]
for pose, bbox in zip(human_poses, bboxes):
cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[0] + bbox[2], bbox[1] + bbox[3]), (255, 0, 0), 2)
for id_kpt, kpt in enumerate(pose):
cv2.circle(frame, (int(kpt[0]), int(kpt[1])), 3, colors[id_kpt], -1)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Human Pose Estimation Demo', frame)
key = cv2.waitKey(delay)
if key == 27:
break
presenter.handleKey(key)
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
if args.architecture_type != 'yolov4' and args.anchors:
log.warning(
'The "--anchors" option works only for "-at==yolov4". Option will be omitted'
)
if args.architecture_type != 'yolov4' and args.masks:
log.warning(
'The "--masks" option works only for "-at==yolov4". Option will be omitted'
)
if args.architecture_type not in ['nanodet', 'nanodet-plus'
] and args.num_classes:
log.warning(
'The "--num_classes" option works only for "-at==nanodet" and "-at==nanodet-plus". Option will be omitted'
)
cap = open_images_capture(args.input, args.loop)
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests,
model_parameters={'input_layouts': args.layout})
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
configuration = {
'resize_type': args.resize_type,
'mean_values': args.mean_values,
'scale_values': args.scale_values,
'reverse_input_channels': args.reverse_input_channels,
'path_to_labels': args.labels,
'confidence_threshold': args.prob_threshold,
'input_size': args.input_size, # The CTPN specific
'num_classes':
args.num_classes, # The NanoDet and NanoDetPlus specific
}
model = DetectionModel.create_model(args.architecture_type, model_adapter,
configuration)
model.log_layers_info()
detector_pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
palette = ColorPalette(len(model.labels) if model.labels else 100)
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
while True:
if detector_pipeline.callback_exceptions:
raise detector_pipeline.callback_exceptions[0]
# Process all completed requests
results = detector_pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(objects) and args.raw_output_message:
print_raw_results(objects, model.labels, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_detections(frame, objects, palette, model.labels,
output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Detection Results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
continue
if detector_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
detector_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
detector_pipeline.await_any()
detector_pipeline.await_all()
if detector_pipeline.callback_exceptions:
raise detector_pipeline.callback_exceptions[0]
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = detector_pipeline.get_result(next_frame_id_to_show)
objects, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(objects) and args.raw_output_message:
print_raw_results(objects, model.labels, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_detections(frame, objects, palette, model.labels,
output_transform)
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Detection Results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
detector_pipeline.preprocess_metrics.get_latency(),
detector_pipeline.inference_metrics.get_latency(),
detector_pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
model = MonoDepthModel(model_adapter)
model.log_layers_info()
pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2], args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {'start_time': start_time})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
depth_map, frame_meta = results
depth_map = apply_color_map(depth_map, output_transform)
start_time = frame_meta['start_time']
presenter.drawGraphs(depth_map)
metrics.update(start_time, depth_map)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(depth_map)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow(DEMO_NAME, depth_map)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
depth_map, frame_meta = results
depth_map = apply_color_map(depth_map, output_transform)
start_time = frame_meta['start_time']
presenter.drawGraphs(depth_map)
metrics.update(start_time, depth_map)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(depth_map)
if not args.no_show:
cv2.imshow(DEMO_NAME, depth_map)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
metrics = PerformanceMetrics()
args = build_argparser().parse_args()
# Plugin initialization for specified device and load extensions library if specified
log.info("Creating Inference Engine")
ie = IECore()
# Read IR
log.info("Loading network files:\n\t{}".format(args.model_pnet))
p_net = ie.read_network(args.model_pnet)
assert len(p_net.input_info.keys()
) == 1, "Pnet supports only single input topologies"
assert len(p_net.outputs) == 2, "Pnet supports two output topologies"
log.info("Loading network files:\n\t{}".format(args.model_rnet))
r_net = ie.read_network(args.model_rnet)
assert len(r_net.input_info.keys()
) == 1, "Rnet supports only single input topologies"
assert len(r_net.outputs) == 2, "Rnet supports two output topologies"
log.info("Loading network files:\n\t{}".format(args.model_onet))
o_net = ie.read_network(args.model_onet)
assert len(o_net.input_info.keys()
) == 1, "Onet supports only single input topologies"
assert len(o_net.outputs) == 3, "Onet supports three output topologies"
log.info("Preparing input blobs")
pnet_input_blob = next(iter(p_net.input_info))
rnet_input_blob = next(iter(r_net.input_info))
onet_input_blob = next(iter(o_net.input_info))
log.info("Preparing output blobs")
for name, blob in p_net.outputs.items():
if blob.shape[1] == 2:
pnet_cls_name = name
elif blob.shape[1] == 4:
pnet_roi_name = name
else:
raise RuntimeError("Unsupported output layer for Pnet")
for name, blob in r_net.outputs.items():
if blob.shape[1] == 2:
rnet_cls_name = name
elif blob.shape[1] == 4:
rnet_roi_name = name
else:
raise RuntimeError("Unsupported output layer for Rnet")
for name, blob in o_net.outputs.items():
if blob.shape[1] == 2:
onet_cls_name = name
elif blob.shape[1] == 4:
onet_roi_name = name
elif blob.shape[1] == 10:
onet_pts_name = name
else:
raise RuntimeError("Unsupported output layer for Onet")
cap = open_images_capture(args.input, args.loop)
next_frame_id = 0
log.info('Starting inference...')
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
presenter = None
video_writer = cv2.VideoWriter()
while True:
start_time = perf_counter()
origin_image = cap.read()
if origin_image is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(origin_image.shape[1] / 4),
round(origin_image.shape[0] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(origin_image.shape[1], origin_image.shape[0])):
raise RuntimeError("Can't open video writer")
next_frame_id += 1
rgb_image = cv2.cvtColor(origin_image, cv2.COLOR_BGR2RGB)
oh, ow, _ = rgb_image.shape
scales = utils.calculate_scales(rgb_image)
# *************************************
# Pnet stage
# *************************************
log.info("Loading Pnet model to the plugin")
t0 = cv2.getTickCount()
pnet_res = []
for scale in scales:
hs = int(oh * scale)
ws = int(ow * scale)
image = preprocess_image(rgb_image, ws, hs)
p_net.reshape({pnet_input_blob:
[1, 3, ws,
hs]}) # Change weidth and height of input blob
exec_pnet = ie.load_network(network=p_net, device_name=args.device)
p_res = exec_pnet.infer(inputs={pnet_input_blob: image})
pnet_res.append(p_res)
image_num = len(scales)
rectangles = []
for i in range(image_num):
roi = pnet_res[i][pnet_roi_name]
cls = pnet_res[i][pnet_cls_name]
_, _, out_h, out_w = cls.shape
out_side = max(out_h, out_w)
rectangle = utils.detect_face_12net(cls[0][1], roi[0], out_side,
1 / scales[i], ow, oh,
score_threshold[0],
iou_threshold[0])
rectangles.extend(rectangle)
rectangles = utils.NMS(rectangles, iou_threshold[1], 'iou')
# Rnet stage
if len(rectangles) > 0:
log.info("Loading Rnet model to the plugin")
r_net.reshape({rnet_input_blob:
[len(rectangles), 3, 24,
24]}) # Change batch size of input blob
exec_rnet = ie.load_network(network=r_net, device_name=args.device)
rnet_input = []
for rectangle in rectangles:
crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]),
int(rectangle[0]):int(rectangle[2])]
crop_img = preprocess_image(crop_img, 24, 24)
rnet_input.extend(crop_img)
rnet_res = exec_rnet.infer(inputs={rnet_input_blob: rnet_input})
roi = rnet_res[rnet_roi_name]
cls = rnet_res[rnet_cls_name]
rectangles = utils.filter_face_24net(cls, roi, rectangles, ow, oh,
score_threshold[1],
iou_threshold[2])
# Onet stage
if len(rectangles) > 0:
log.info("Loading Onet model to the plugin")
o_net.reshape({onet_input_blob:
[len(rectangles), 3, 48,
48]}) # Change batch size of input blob
exec_onet = ie.load_network(network=o_net, device_name=args.device)
onet_input = []
for rectangle in rectangles:
crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]),
int(rectangle[0]):int(rectangle[2])]
crop_img = preprocess_image(crop_img, 48, 48)
onet_input.extend(crop_img)
onet_res = exec_onet.infer(inputs={onet_input_blob: onet_input})
roi = onet_res[onet_roi_name]
cls = onet_res[onet_cls_name]
pts = onet_res[onet_pts_name]
rectangles = utils.filter_face_48net(cls, roi, pts, rectangles, ow,
oh, score_threshold[2],
iou_threshold[3])
# display results
for rectangle in rectangles:
# Draw detected boxes
cv2.putText(origin_image,
'confidence: {:.2f}'.format(rectangle[4]),
(int(rectangle[0]), int(rectangle[1])),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0))
cv2.rectangle(origin_image, (int(rectangle[0]), int(rectangle[1])),
(int(rectangle[2]), int(rectangle[3])), (255, 0, 0),
1)
# Draw landmarks
for i in range(5, 15, 2):
cv2.circle(origin_image,
(int(rectangle[i + 0]), int(rectangle[i + 1])), 2,
(0, 255, 0))
infer_time = (cv2.getTickCount() -
t0) / cv2.getTickFrequency() # Record infer time
cv2.putText(origin_image,
'summary: {:.1f} FPS'.format(1.0 / infer_time), (5, 15),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 200))
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id
<= args.output_limit - 1):
video_writer.write(origin_image)
if not args.no_show:
cv2.imshow('MTCNN Results', origin_image)
key = cv2.waitKey(1)
if key in {ord('q'), ord('Q'), 27}:
break
presenter.handleKey(key)
metrics.update(start_time, origin_image)
metrics.print_total()
def main():
parser = argparse.ArgumentParser(description='Whiteboard inpainting demo')
parser.add_argument('-i', '--input', required=True,
help='Required. Path to a video file or a device node of a web-camera.')
parser.add_argument('--loop', default=False, action='store_true',
help='Optional. Enable reading the input in a loop.')
parser.add_argument('-o', '--output', required=False,
help='Optional. Name of the output file(s) to save.')
parser.add_argument('-limit', '--output_limit', required=False, default=1000, type=int,
help='Optional. Number of frames to store in output. '
'If 0 is set, all frames are stored.')
parser.add_argument('-m_i', '--m_instance_segmentation', type=str, required=False,
help='Required. Path to the instance segmentation model.')
parser.add_argument('-m_s', '--m_semantic_segmentation', type=str, required=False,
help='Required. Path to the semantic segmentation model.')
parser.add_argument('-t', '--threshold', type=float, default=0.6,
help='Optional. Threshold for person instance segmentation model.')
parser.add_argument('--no_show', help="Optional. Don't show output.", action='store_true')
parser.add_argument('-d', '--device', type=str, default='CPU',
help='Optional. Specify a target device to infer on. CPU, GPU, HDDL or MYRIAD is '
'acceptable. The demo will look for a suitable plugin for the device specified.')
parser.add_argument('-l', '--cpu_extension', type=str, default=None,
help='MKLDNN (CPU)-targeted custom layers. Absolute \
path to a shared library with the kernels impl.')
parser.add_argument('-u', '--utilization_monitors', default='', type=str,
help='Optional. List of monitors to show initially.')
args = parser.parse_args()
cap = open_images_capture(args.input, args.loop)
if cap.get_type() not in ('VIDEO', 'CAMERA'):
raise RuntimeError("The input should be a video file or a numeric camera ID")
if bool(args.m_instance_segmentation) == bool(args.m_semantic_segmentation):
raise ValueError('Set up exactly one of segmentation models: '
'--m_instance_segmentation or --m_semantic_segmentation')
labels_dir = Path(__file__).resolve().parents[3] / 'data/dataset_classes'
mouse = MouseClick()
if not args.no_show:
cv2.namedWindow(WINNAME)
cv2.setMouseCallback(WINNAME, mouse.get_points)
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
model_path = args.m_instance_segmentation if args.m_instance_segmentation else args.m_semantic_segmentation
log.info('Reading model {}'.format(model_path))
if args.m_instance_segmentation:
labels_file = str(labels_dir / 'coco_80cl_bkgr.txt')
segmentation = MaskRCNN(core, args.m_instance_segmentation, labels_file,
args.threshold, args.device, args.cpu_extension)
elif args.m_semantic_segmentation:
labels_file = str(labels_dir / 'cityscapes_19cl_bkgr.txt')
segmentation = SemanticSegmentation(core, args.m_semantic_segmentation, labels_file,
args.threshold, args.device, args.cpu_extension)
log.info('The model {} is loaded to {}'.format(model_path, args.device))
metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
black_board = False
frame_number = 0
key = -1
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
out_frame_size = (frame.shape[1], frame.shape[0] * 2)
output_frame = np.full((frame.shape[0], frame.shape[1], 3), 255, dtype='uint8')
presenter = monitors.Presenter(args.utilization_monitors, 20,
(out_frame_size[0] // 4, out_frame_size[1] // 16))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), out_frame_size):
raise RuntimeError("Can't open video writer")
while frame is not None:
mask = None
detections = segmentation.get_detections([frame])
expand_mask(detections, frame.shape[1] // 27)
if len(detections[0]) > 0:
mask = detections[0][0][2]
for i in range(1, len(detections[0])):
mask = cv2.bitwise_or(mask, detections[0][i][2])
if mask is not None:
mask = np.stack([mask, mask, mask], axis=-1)
else:
mask = np.zeros(frame.shape, dtype='uint8')
clear_frame = remove_background(frame, invert_colors=not black_board)
output_frame = np.where(mask, output_frame, clear_frame)
merged_frame = np.vstack([frame, output_frame])
merged_frame = cv2.resize(merged_frame, out_frame_size)
metrics.update(start_time, merged_frame)
if video_writer.isOpened() and (args.output_limit <= 0 or frame_number <= args.output_limit-1):
video_writer.write(merged_frame)
presenter.drawGraphs(merged_frame)
if not args.no_show:
cv2.imshow(WINNAME, merged_frame)
key = check_pressed_keys(key)
if key == 27: # 'Esc'
break
if key == ord('i'): # catch pressing of key 'i'
black_board = not black_board
output_frame = 255 - output_frame
else:
presenter.handleKey(key)
if mouse.crop_available:
x0, x1 = min(mouse.points[0][0], mouse.points[1][0]), \
max(mouse.points[0][0], mouse.points[1][0])
y0, y1 = min(mouse.points[0][1], mouse.points[1][1]), \
max(mouse.points[0][1], mouse.points[1][1])
x1, y1 = min(x1, output_frame.shape[1] - 1), min(y1, output_frame.shape[0] - 1)
board = output_frame[y0: y1, x0: x1, :]
if board.shape[0] > 0 and board.shape[1] > 0:
cv2.namedWindow('Board', cv2.WINDOW_KEEPRATIO)
cv2.imshow('Board', board)
frame_number += 1
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
place_recognition = PlaceRecognition(args.model, args.device,
args.gallery_folder,
args.gallery_size)
compute_embeddings_times = []
search_in_gallery_times = []
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 0)
video_writer = cv2.VideoWriter()
metrics = PerformanceMetrics()
while True:
start_time = perf_counter()
frame = cap.read()
if frame is None:
if frames_processed == 0:
raise ValueError("Can't read an image from the input")
break
elapsed, probe_embedding = time_elapsed(
place_recognition.compute_embedding, frame)
compute_embeddings_times.append(elapsed)
elapsed, (sorted_indexes, distances) = time_elapsed(
place_recognition.search_in_gallery, probe_embedding)
search_in_gallery_times.append(elapsed)
image, key = visualize(
frame, [str(place_recognition.impaths[i]) for i in sorted_indexes],
distances[sorted_indexes],
place_recognition.input_size,
np.mean(compute_embeddings_times),
np.mean(search_in_gallery_times),
imshow_delay=3,
presenter=presenter,
no_show=args.no_show)
metrics.update(start_time)
if frames_processed == 0:
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(image.shape[1], image.shape[0])):
raise RuntimeError("Can't open video writer")
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(image)
if key == 27:
break
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
if args.labels:
with open(args.labels) as f:
labels = [line.strip() for line in f]
else:
labels = None
ie = IECore()
if 'MYRIAD' in args.device:
myriad_config = {'VPU_HW_STAGES_OPTIMIZATION': 'YES'}
ie.set_config(myriad_config, 'MYRIAD')
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, 'CPU')
decoder_target_device = 'CPU'
if args.device != 'CPU':
encoder_target_device = args.device
else:
encoder_target_device = decoder_target_device
encoder_xml = args.m_encoder
encoder_bin = args.m_encoder.replace('.xml', '.bin')
models = [
IEModel(encoder_xml,
encoder_bin,
ie,
encoder_target_device,
num_requests=(3 if args.device == 'MYRIAD' else 1))
]
if args.architecture_type == 'en-de':
if args.m_decoder is None:
raise RuntimeError(
'No decoder for encoder-decoder model type (-m_de) provided')
decoder_xml = args.m_decoder
decoder_bin = args.m_decoder.replace('.xml', '.bin')
models.append(
IEModel(decoder_xml,
decoder_bin,
ie,
decoder_target_device,
num_requests=2))
seq_size = models[1].input_size[1]
elif args.architecture_type == 'en-mean':
models.append(DummyDecoder(num_requests=2))
seq_size = args.decoder_seq_size
elif args.architecture_type == 'i3d-rgb':
seq_size = models[0].input_size[2]
presenter = monitors.Presenter(args.utilization_monitors, 70)
result_presenter = ResultRenderer(
no_show=args.no_show,
presenter=presenter,
output=args.output,
limit=args.output_limit,
labels=labels,
label_smoothing_window=args.label_smoothing)
cap = open_images_capture(args.input, args.loop)
run_pipeline(cap,
args.architecture_type,
models,
result_presenter.render_frame,
seq_size=seq_size,
fps=cap.fps())
print(presenter.reportMeans())
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
delay = int(cap.get_type() in {'VIDEO', 'CAMERA'})
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
config = {
'mean_values': args.mean_values,
'scale_values': args.scale_values,
'reverse_input_channels': args.reverse_input_channels,
'topk': args.topk,
'path_to_labels': args.labels
}
model = Classification(model_adapter, config)
model.log_layers_info()
async_pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
ESC_KEY = 27
key = -1
while True:
if async_pipeline.callback_exceptions:
raise async_pipeline.callback_exceptions[0]
# Process all completed requests
results = async_pipeline.get_result(next_frame_id_to_show)
if results:
classifications, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if args.raw_output_message:
print_raw_results(classifications, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_labels(frame, classifications, output_transform)
if delay or args.no_show:
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Classification Results', frame)
key = cv2.waitKey(delay)
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
continue
if async_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
async_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
async_pipeline.await_any()
async_pipeline.await_all()
if key not in {ord('q'), ord('Q'), ESC_KEY}:
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show,
next_frame_id):
results = async_pipeline.get_result(next_frame_id_to_show)
while results is None:
results = async_pipeline.get_result(next_frame_id_to_show)
classifications, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if args.raw_output_message:
print_raw_results(classifications, next_frame_id_to_show)
presenter.drawGraphs(frame)
rendering_start_time = perf_counter()
frame = draw_labels(frame, classifications, output_transform)
if delay or args.no_show:
render_metrics.update(rendering_start_time)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Classification Results', frame)
key = cv2.waitKey(delay)
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
if delay or args.no_show:
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
async_pipeline.preprocess_metrics.get_latency(),
async_pipeline.inference_metrics.get_latency(),
async_pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
# Plugin initialization for specified device and load extensions library if specified.
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, 'CPU')
# Read IR
log.info('Reading Mask-RCNN model {}'.format(args.mask_rcnn_model))
mask_rcnn_net = ie.read_network(args.mask_rcnn_model)
model_required_inputs = {'image'}
if set(mask_rcnn_net.input_info) == model_required_inputs:
required_output_keys = {'boxes', 'labels', 'masks', 'text_features.0'}
n, c, h, w = mask_rcnn_net.input_info['image'].input_data.shape
assert n == 1, 'Only batch 1 is supported by the demo application'
else:
raise RuntimeError(
'Demo supports only topologies with the following input keys: '
f'{model_required_inputs}.')
assert required_output_keys.issubset(mask_rcnn_net.outputs.keys()), \
f'Demo supports only topologies with the following output keys: {required_output_keys}' \
f'Found: {mask_rcnn_net.outputs.keys()}.'
log.info('Reading Text Recognition Encoder model {}'.format(
args.text_enc_model))
text_enc_net = ie.read_network(args.text_enc_model)
log.info('Reading Text Recognition Decoder model {}'.format(
args.text_dec_model))
text_dec_net = ie.read_network(args.text_dec_model)
mask_rcnn_exec_net = ie.load_network(network=mask_rcnn_net,
device_name=args.device,
num_requests=2)
log.info('The Mask-RCNN model {} is loaded to {}'.format(
args.mask_rcnn_model, args.device))
text_enc_exec_net = ie.load_network(network=text_enc_net,
device_name=args.device)
log.info('The Text Recognition Encoder model {} is loaded to {}'.format(
args.text_enc_model, args.device))
text_dec_exec_net = ie.load_network(network=text_dec_net,
device_name=args.device)
log.info('The Text Recognition Decoder model {} is loaded to {}'.format(
args.text_dec_model, args.device))
hidden_shape = text_dec_net.input_info[
args.trd_input_prev_hidden].input_data.shape
del mask_rcnn_net
del text_enc_net
del text_dec_net
if args.no_track:
tracker = None
else:
tracker = StaticIOUTracker()
if args.delay:
delay = args.delay
else:
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
visualizer = Visualizer(['__background__', 'text'],
show_boxes=args.show_boxes,
show_scores=args.show_scores)
frames_processed = 0
metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
presenter = monitors.Presenter(args.utilization_monitors, 45,
(frame.shape[1] // 4, frame.shape[0] // 8))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while frame is not None:
if not args.keep_aspect_ratio:
# Resize the image to a target size.
scale_x = w / frame.shape[1]
scale_y = h / frame.shape[0]
input_image = cv2.resize(frame, (w, h))
else:
# Resize the image to keep the same aspect ratio and to fit it to a window of a target size.
scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1])
input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y)
input_image_size = input_image.shape[:2]
input_image = np.pad(input_image,
((0, h - input_image_size[0]),
(0, w - input_image_size[1]), (0, 0)),
mode='constant',
constant_values=0)
# Change data layout from HWC to CHW.
input_image = input_image.transpose((2, 0, 1))
input_image = input_image.reshape((n, c, h, w)).astype(np.float32)
# Run the net.
outputs = mask_rcnn_exec_net.infer({'image': input_image})
# Parse detection results of the current request
boxes = outputs['boxes'][:, :4]
scores = outputs['boxes'][:, 4]
classes = outputs['labels'].astype(np.uint32)
raw_masks = outputs['masks']
text_features = outputs['text_features.0']
# Filter out detections with low confidence.
detections_filter = scores > args.prob_threshold
scores = scores[detections_filter]
classes = classes[detections_filter]
boxes = boxes[detections_filter]
raw_masks = raw_masks[detections_filter]
text_features = text_features[detections_filter]
boxes[:, 0::2] /= scale_x
boxes[:, 1::2] /= scale_y
masks = []
for box, cls, raw_mask in zip(boxes, classes, raw_masks):
mask = segm_postprocess(box, raw_mask, frame.shape[0],
frame.shape[1])
masks.append(mask)
texts = []
for feature in text_features:
feature = text_enc_exec_net.infer({'input': feature})['output']
feature = np.reshape(feature,
(feature.shape[0], feature.shape[1], -1))
feature = np.transpose(feature, (0, 2, 1))
hidden = np.zeros(hidden_shape)
prev_symbol_index = np.ones((1, )) * SOS_INDEX
text = ''
text_confidence = 1.0
for i in range(MAX_SEQ_LEN):
decoder_output = text_dec_exec_net.infer({
args.trd_input_prev_symbol:
prev_symbol_index,
args.trd_input_prev_hidden:
hidden,
args.trd_input_encoder_outputs:
feature
})
symbols_distr = decoder_output[args.trd_output_symbols_distr]
symbols_distr_softmaxed = softmax(symbols_distr, axis=1)[0]
prev_symbol_index = int(np.argmax(symbols_distr, axis=1))
text_confidence *= symbols_distr_softmaxed[prev_symbol_index]
if prev_symbol_index == EOS_INDEX:
break
text += args.alphabet[prev_symbol_index]
hidden = decoder_output[args.trd_output_cur_hidden]
texts.append(text if text_confidence >= args.tr_threshold else '')
if len(boxes) and args.raw_output_message:
log.debug(
' -------------------------- Frame # {} -------------------------- '
.format(frames_processed))
log.debug(
' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX '
)
for box, cls, score, mask in zip(boxes, classes, scores, masks):
log.debug(
'{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} '
.format(cls, score, *box))
# Get instance track IDs.
masks_tracks_ids = None
if tracker is not None:
masks_tracks_ids = tracker(masks, classes)
presenter.drawGraphs(frame)
# Visualize masks.
frame = visualizer(frame, boxes, classes, scores, masks, texts,
masks_tracks_ids)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
# Show resulting image.
cv2.imshow('Results', frame)
if not args.no_show:
key = cv2.waitKey(delay)
esc_code = 27
if key == esc_code:
break
presenter.handleKey(key)
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
cv2.destroyAllWindows()
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
target_bgr = open_images_capture(args.target_bgr, loop=True) if args.target_bgr else None
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config,
max_num_requests=args.num_infer_requests)
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
labels = ['__background__', 'person'] if args.labels is None else load_labels(args.labels)
assert len(labels), 'The file with class labels is empty'
configuration = {
'confidence_threshold': args.prob_threshold,
'resize_type': args.resize_type
}
model, need_bgr_input = get_model(model_adapter, configuration, args)
input_bgr = open_images_capture(args.background, False).read() if need_bgr_input else None
person_id = -1
for i, label in enumerate(labels):
if label == 'person':
person_id = i
break
assert person_id >= 0, 'Person class did not find in labels list.'
model.log_layers_info()
pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
bgr = target_bgr.read() if target_bgr is not None else None
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2], args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), tuple(output_resolution)):
raise RuntimeError("Can't open video writer")
# Submit for inference
data = {'src': frame, 'bgr': input_bgr} if input_bgr is not None else frame
pipeline.submit_data(data, next_frame_id, {'frame': frame, 'start_time': start_time})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_results(frame, objects, output_resolution, bgr, person_id,
args.blur_bgr, args.show_with_original_frame)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Background subtraction results', frame)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show, model.labels)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_results(frame, objects, output_resolution, bgr, person_id,
args.blur_bgr, args.show_with_original_frame)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Background subtraction results', frame)
cv2.waitKey(1)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
pipeline.preprocess_metrics.get_latency(),
pipeline.inference_metrics.get_latency(),
pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
log.basicConfig(format='[ %(levelname)s ] %(message)s',
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
# Plugin initialization for specified device and load extensions library if specified.
log.info('Creating Inference Engine...')
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, 'CPU')
# Read IR
log.info('Loading network')
net = ie.read_network(args.model, args.model.with_suffix('.bin'))
image_input, image_info_input, (
n, c, h, w), model_type, postprocessor = check_model(net)
args.no_keep_aspect_ratio = model_type == 'yolact' or args.no_keep_aspect_ratio
log.info('Loading IR to the plugin...')
exec_net = ie.load_network(network=net,
device_name=args.device,
num_requests=2)
cap = open_images_capture(args.input, args.loop)
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
if args.no_track:
tracker = None
else:
tracker = StaticIOUTracker()
with open(args.labels, 'rt') as labels_file:
class_labels = labels_file.read().splitlines()
if args.delay:
delay = args.delay
else:
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
frames_processed = 0
out_frame_size = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 45,
(round(out_frame_size[0] / 4), round(out_frame_size[1] / 8)))
visualizer = Visualizer(class_labels,
show_boxes=args.show_boxes,
show_scores=args.show_scores)
video_writer = cv2.VideoWriter()
if args.output and not video_writer.open(args.output,
cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), out_frame_size):
raise RuntimeError("Can't open video writer")
render_time = 0
log.info('Starting inference...')
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
while frame is not None:
if args.no_keep_aspect_ratio:
# Resize the image to a target size.
scale_x = w / frame.shape[1]
scale_y = h / frame.shape[0]
input_image = cv2.resize(frame, (w, h))
else:
# Resize the image to keep the same aspect ratio and to fit it to a window of a target size.
scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1])
input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y)
input_image_size = input_image.shape[:2]
input_image = np.pad(input_image,
((0, h - input_image_size[0]),
(0, w - input_image_size[1]), (0, 0)),
mode='constant',
constant_values=0)
# Change data layout from HWC to CHW.
input_image = input_image.transpose((2, 0, 1))
input_image = input_image.reshape((n, c, h, w)).astype(np.float32)
input_image_info = np.asarray(
[[input_image_size[0], input_image_size[1], 1]], dtype=np.float32)
# Run the net.
inf_start = time.time()
feed_dict = {image_input: input_image}
if image_info_input:
feed_dict[image_info_input] = input_image_info
outputs = exec_net.infer(feed_dict)
inf_end = time.time()
det_time = inf_end - inf_start
# Parse detection results of the current request
scores, classes, boxes, masks = postprocessor(outputs, scale_x,
scale_y,
*frame.shape[:2], h, w,
args.prob_threshold)
render_start = time.time()
if len(boxes) and args.raw_output_message:
log.info('Detected boxes:')
log.info(
' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX '
)
for box, cls, score, mask in zip(boxes, classes, scores, masks):
log.info(
'{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} '
.format(cls, score, *box))
# Get instance track IDs.
masks_tracks_ids = None
if tracker is not None:
masks_tracks_ids = tracker(masks, classes)
# Visualize masks.
frame = visualizer(frame, boxes, classes, scores, presenter, masks,
masks_tracks_ids)
# Draw performance stats.
inf_time_message = 'Inference time: {:.3f} ms'.format(det_time * 1000)
render_time_message = 'OpenCV rendering time: {:.3f} ms'.format(
render_time * 1000)
cv2.putText(frame, inf_time_message, (15, 15),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1)
cv2.putText(frame, render_time_message, (15, 30),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
# Print performance counters.
if args.perf_counts:
perf_counts = exec_net.requests[0].get_perf_counts()
log.info('Performance counters:')
print('{:<70} {:<15} {:<15} {:<15} {:<10}'.format(
'name', 'layer_type', 'exet_type', 'status', 'real_time, us'))
for layer, stats in perf_counts.items():
print('{:<70} {:<15} {:<15} {:<15} {:<10}'.format(
layer, stats['layer_type'], stats['exec_type'],
stats['status'], stats['real_time']))
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
# Show resulting image.
cv2.imshow('Results', frame)
render_end = time.time()
render_time = render_end - render_start
if not args.no_show:
key = cv2.waitKey(delay)
esc_code = 27
if key == esc_code:
break
presenter.handleKey(key)
frame = cap.read()
print(presenter.reportMeans())
cv2.destroyAllWindows()
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
with open(args.labels, 'rt') as labels_file:
class_labels = labels_file.read().splitlines()
assert len(class_labels), 'The file with class labels is empty'
# Plugin initialization for specified device and load extensions library if specified.
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, 'CPU')
# Read IR
log.info('Reading model {}'.format(args.model))
net = ie.read_network(args.model, args.model.with_suffix('.bin'))
image_input, image_info_input, (
n, c, h, w), model_type, postprocessor = check_model(net)
args.no_keep_aspect_ratio = model_type == 'yolact' or args.no_keep_aspect_ratio
exec_net = ie.load_network(network=net,
device_name=args.device,
num_requests=2)
log.info('The model {} is loaded to {}'.format(args.model, args.device))
if args.no_track:
tracker = None
else:
tracker = StaticIOUTracker()
if args.delay:
delay = args.delay
else:
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
frames_processed = 0
metrics = PerformanceMetrics()
visualizer = Visualizer(class_labels,
show_boxes=args.show_boxes,
show_scores=args.show_scores)
video_writer = cv2.VideoWriter()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
out_frame_size = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 45,
(round(out_frame_size[0] / 4), round(out_frame_size[1] / 8)))
if args.output and not video_writer.open(args.output,
cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), out_frame_size):
raise RuntimeError("Can't open video writer")
while frame is not None:
if args.no_keep_aspect_ratio:
# Resize the image to a target size.
scale_x = w / frame.shape[1]
scale_y = h / frame.shape[0]
input_image = cv2.resize(frame, (w, h))
else:
# Resize the image to keep the same aspect ratio and to fit it to a window of a target size.
scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1])
input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y)
input_image_size = input_image.shape[:2]
input_image = np.pad(input_image,
((0, h - input_image_size[0]),
(0, w - input_image_size[1]), (0, 0)),
mode='constant',
constant_values=0)
# Change data layout from HWC to CHW.
input_image = input_image.transpose((2, 0, 1))
input_image = input_image.reshape((n, c, h, w)).astype(np.float32)
input_image_info = np.asarray(
[[input_image_size[0], input_image_size[1], 1]], dtype=np.float32)
# Run the net.
feed_dict = {image_input: input_image}
if image_info_input:
feed_dict[image_info_input] = input_image_info
outputs = exec_net.infer(feed_dict)
# Parse detection results of the current request
scores, classes, boxes, masks = postprocessor(outputs, scale_x,
scale_y,
*frame.shape[:2], h, w,
args.prob_threshold)
if len(boxes) and args.raw_output_message:
log.debug(
' -------------------------- Frame # {} -------------------------- '
.format(frames_processed))
log.debug(
' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX '
)
for box, cls, score, mask in zip(boxes, classes, scores, masks):
log.debug(
'{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} '
.format(cls, score, *box))
# Get instance track IDs.
masks_tracks_ids = None
if tracker is not None:
masks_tracks_ids = tracker(masks, classes)
# Visualize masks.
frame = visualizer(frame, boxes, classes, scores, presenter, masks,
masks_tracks_ids)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
# Show resulting image.
cv2.imshow('Results', frame)
if not args.no_show:
key = cv2.waitKey(delay)
esc_code = 27
if key == esc_code:
break
presenter.handleKey(key)
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
cv2.destroyAllWindows()
def main():
args = build_argparser().parse_args()
log.info('Initializing Inference Engine...')
ie = IECore()
plugin_config = get_plugin_configs(args.device, args.num_streams,
args.num_threads)
log.info('Loading network...')
model = get_model(ie, args)
detector_pipeline = AsyncPipeline(ie,
model,
plugin_config,
device=args.device,
max_num_requests=args.num_infer_requests)
cap = open_images_capture(args.input, args.loop)
next_frame_id = 0
next_frame_id_to_show = 0
log.info('Starting inference...')
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
palette = ColorPalette(len(model.labels) if model.labels else 100)
metrics = PerformanceMetrics()
presenter = None
video_writer = cv2.VideoWriter()
while True:
if detector_pipeline.callback_exceptions:
raise detector_pipeline.callback_exceptions[0]
# Process all completed requests
results = detector_pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(objects) and args.raw_output_message:
print_raw_results(frame.shape[:2], objects, model.labels,
args.prob_threshold)
presenter.drawGraphs(frame)
frame = draw_detections(frame, objects, palette, model.labels,
args.prob_threshold)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Detection Results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
next_frame_id_to_show += 1
continue
if detector_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(frame.shape[1] / 4), round(frame.shape[0] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), (frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
# Submit for inference
detector_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
detector_pipeline.await_any()
detector_pipeline.await_all()
# Process completed requests
while detector_pipeline.has_completed_request():
results = detector_pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(objects) and args.raw_output_message:
print_raw_results(frame.shape[:2], objects, model.labels,
args.prob_threshold)
presenter.drawGraphs(frame)
frame = draw_detections(frame, objects, palette, model.labels,
args.prob_threshold)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Detection Results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
next_frame_id_to_show += 1
else:
break
metrics.print_total()
print(presenter.reportMeans())
default=None)
args.add_argument('--fx',
type=np.float32,
default=-1,
help='Optional. Camera focal length.')
args.add_argument('--no_show',
help='Optional. Do not display output.',
action='store_true')
args.add_argument("-u",
"--utilization_monitors",
default='',
type=str,
help="Optional. List of monitors to show initially.")
args = parser.parse_args()
cap = open_images_capture(args.input, args.loop)
stride = 8
inference_engine = InferenceEngine(args.model, args.device, stride)
canvas_3d = np.zeros((720, 1280, 3), dtype=np.uint8)
plotter = Plotter3d(canvas_3d.shape[:2])
canvas_3d_window_name = 'Canvas 3D'
if not args.no_show:
cv2.namedWindow(canvas_3d_window_name)
cv2.setMouseCallback(canvas_3d_window_name, Plotter3d.mouse_callback)
file_path = args.extrinsics_path
if file_path is None:
file_path = Path(__file__).parent / 'data/extrinsics.json'
with open(file_path, 'r') as f:
extrinsics = json.load(f)
def main():
args = build_argparser().parse_args()
log.info('Initializing Inference Engine...')
ie = IECore()
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
cap = open_images_capture(args.input, args.loop)
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
log.info('Loading network...')
model = Deblurring(ie, args.model, frame.shape)
pipeline = AsyncPipeline(ie,
model,
plugin_config,
device=args.device,
max_num_requests=args.num_infer_requests)
log.info('Starting inference...')
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
pipeline.submit_data(frame, 0, {'frame': frame, 'start_time': start_time})
next_frame_id = 1
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(frame.shape[1] / 4), round(frame.shape[0] / 8)))
video_writer = cv2.VideoWriter()
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(2 * frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
break
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
result_frame, frame_meta = results
input_frame = frame_meta['frame']
start_time = frame_meta['start_time']
if input_frame.shape != result_frame.shape:
input_frame = cv2.resize(
input_frame,
(result_frame.shape[1], result_frame.shape[0]))
final_image = cv2.hconcat([input_frame, result_frame])
presenter.drawGraphs(final_image)
metrics.update(start_time, final_image)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(final_image)
if not args.no_show:
cv2.imshow('Deblurring Results', final_image)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
presenter.handleKey(key)
next_frame_id_to_show += 1
pipeline.await_all()
# Process completed requests
while pipeline.has_completed_request():
results = pipeline.get_result(next_frame_id_to_show)
if results:
result_frame, frame_meta = results
input_frame = frame_meta['frame']
start_time = frame_meta['start_time']
if input_frame.shape != result_frame.shape:
input_frame = cv2.resize(
input_frame,
(result_frame.shape[1], result_frame.shape[0]))
final_image = cv2.hconcat([input_frame, result_frame])
presenter.drawGraphs(final_image)
metrics.update(start_time, final_image)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(final_image)
if not args.no_show:
cv2.imshow('Deblurring Results', final_image)
key = cv2.waitKey(1)
next_frame_id_to_show += 1
else:
break
metrics.print_total()
print(presenter.reportMeans())
def main(args):
cap = open_images_capture(args.input, args.loop)
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
log.info('Reading model {}'.format(args.model))
model = core.read_model(args.model)
input_tensor_name = 'data_l'
input_shape = model.input(input_tensor_name).shape
assert input_shape[1] == 1, "Expected model input shape with 1 channel"
inputs = {}
for input in model.inputs:
inputs[input.get_any_name()] = np.zeros(input.shape)
assert len(model.outputs) == 1, "Expected number of outputs is equal 1"
compiled_model = core.compile_model(model, device_name=args.device)
output_tensor = compiled_model.outputs[0]
infer_request = compiled_model.create_infer_request()
log.info('The model {} is loaded to {}'.format(args.model, args.device))
_, _, h_in, w_in = input_shape
frames_processed = 0
imshow_size = (640, 480)
graph_size = (imshow_size[0] // 2, imshow_size[1] // 4)
presenter = monitors.Presenter(args.utilization_monitors,
imshow_size[1] * 2 - graph_size[1],
graph_size)
metrics = PerformanceMetrics()
video_writer = cv.VideoWriter()
if args.output and not video_writer.open(
args.output, cv.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(imshow_size[0] * 2, imshow_size[1] * 2)):
raise RuntimeError("Can't open video writer")
start_time = perf_counter()
original_frame = cap.read()
if original_frame is None:
raise RuntimeError("Can't read an image from the input")
while original_frame is not None:
(h_orig, w_orig) = original_frame.shape[:2]
if original_frame.shape[2] > 1:
frame = cv.cvtColor(cv.cvtColor(original_frame, cv.COLOR_BGR2GRAY),
cv.COLOR_GRAY2RGB)
else:
frame = cv.cvtColor(original_frame, cv.COLOR_GRAY2RGB)
img_rgb = frame.astype(np.float32) / 255
img_lab = cv.cvtColor(img_rgb, cv.COLOR_RGB2Lab)
img_l_rs = cv.resize(img_lab.copy(), (w_in, h_in))[:, :, 0]
inputs[input_tensor_name] = np.expand_dims(img_l_rs, axis=[0, 1])
res = infer_request.infer(inputs)[output_tensor]
update_res = np.squeeze(res)
out = update_res.transpose((1, 2, 0))
out = cv.resize(out, (w_orig, h_orig))
img_lab_out = np.concatenate((img_lab[:, :, 0][:, :, np.newaxis], out),
axis=2)
img_bgr_out = np.clip(cv.cvtColor(img_lab_out, cv.COLOR_Lab2BGR), 0, 1)
original_image = cv.resize(original_frame, imshow_size)
grayscale_image = cv.resize(frame, imshow_size)
colorize_image = (cv.resize(img_bgr_out, imshow_size) * 255).astype(
np.uint8)
lab_image = cv.resize(img_lab_out, imshow_size).astype(np.uint8)
original_image = cv.putText(original_image, 'Original', (25, 50),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2,
cv.LINE_AA)
grayscale_image = cv.putText(grayscale_image, 'Grayscale', (25, 50),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255),
2, cv.LINE_AA)
colorize_image = cv.putText(colorize_image, 'Colorize', (25, 50),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2,
cv.LINE_AA)
lab_image = cv.putText(lab_image, 'LAB interpretation', (25, 50),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2,
cv.LINE_AA)
ir_image = [
cv.hconcat([original_image, grayscale_image]),
cv.hconcat([lab_image, colorize_image])
]
final_image = cv.vconcat(ir_image)
metrics.update(start_time, final_image)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(final_image)
presenter.drawGraphs(final_image)
if not args.no_show:
cv.imshow('Colorization Demo', final_image)
key = cv.waitKey(1)
if key in {ord("q"), ord("Q"), 27}:
break
presenter.handleKey(key)
start_time = perf_counter()
original_frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
if args.adapter == 'openvino':
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
model_adapter = OpenvinoAdapter(
create_core(),
args.model,
device=args.device,
plugin_config=plugin_config,
max_num_requests=args.num_infer_requests,
model_parameters={'input_layouts': args.layout})
elif args.adapter == 'ovms':
model_adapter = OVMSAdapter(args.model)
model = SegmentationModel.create_model(args.architecture_type,
model_adapter,
{'path_to_labels': args.labels})
if args.architecture_type == 'segmentation':
visualizer = SegmentationVisualizer(args.colors)
if args.architecture_type == 'salient_object_detection':
visualizer = SaliencyMapVisualizer()
model.log_layers_info()
pipeline = AsyncPipeline(model)
next_frame_id = 0
next_frame_id_to_show = 0
metrics = PerformanceMetrics()
render_metrics = PerformanceMetrics()
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
only_masks = args.only_masks
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show, model.labels)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_segmentation(frame, objects, visualizer,
output_transform, only_masks)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Segmentation Results', frame)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
if key == 9:
only_masks = not only_masks
presenter.handleKey(key)
pipeline.await_all()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
objects, frame_meta = results
if args.raw_output_message:
print_raw_results(objects, next_frame_id_to_show, model.labels)
frame = frame_meta['frame']
start_time = frame_meta['start_time']
rendering_start_time = perf_counter()
frame = render_segmentation(frame, objects, visualizer,
output_transform, only_masks)
render_metrics.update(rendering_start_time)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Segmentation Results', frame)
key = cv2.waitKey(1)
metrics.log_total()
log_latency_per_stage(cap.reader_metrics.get_latency(),
pipeline.preprocess_metrics.get_latency(),
pipeline.inference_metrics.get_latency(),
pipeline.postprocess_metrics.get_latency(),
render_metrics.get_latency())
for rep in presenter.reportMeans():
log.info(rep)
def main():
args = build_argparser().parse_args()
metrics = PerformanceMetrics()
log.info('Initializing Inference Engine...')
ie = IECore()
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
cap = open_images_capture(args.input, args.loop)
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
log.info('Loading network...')
model = get_model(ie, args, frame.shape[1] / frame.shape[0])
hpe_pipeline = AsyncPipeline(ie,
model,
plugin_config,
device=args.device,
max_num_requests=args.num_infer_requests)
log.info('Starting inference...')
hpe_pipeline.submit_data(frame, 0, {
'frame': frame,
'start_time': start_time
})
next_frame_id = 1
next_frame_id_to_show = 0
output_transform = models.OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
video_writer = cv2.VideoWriter()
if args.output and not video_writer.open(args.output,
cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
while True:
if hpe_pipeline.callback_exceptions:
raise hpe_pipeline.callback_exceptions[0]
# Process all completed requests
results = hpe_pipeline.get_result(next_frame_id_to_show)
if results:
(poses, scores), frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(poses) and args.raw_output_message:
print_raw_results(poses, scores)
presenter.drawGraphs(frame)
frame = draw_poses(frame, poses, args.prob_threshold,
output_transform)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Pose estimation results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
continue
if hpe_pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
break
# Submit for inference
hpe_pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
hpe_pipeline.await_any()
hpe_pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = hpe_pipeline.get_result(next_frame_id_to_show)
while results is None:
results = hpe_pipeline.get_result(next_frame_id_to_show)
(poses, scores), frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(poses) and args.raw_output_message:
print_raw_results(poses, scores)
presenter.drawGraphs(frame)
frame = draw_poses(frame, poses, args.prob_threshold, output_transform)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Pose estimation results', frame)
key = cv2.waitKey(1)
ESC_KEY = 27
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
presenter.handleKey(key)
metrics.print_total()
print(presenter.reportMeans())
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
frame_processor = FrameProcessor(args)
frame_num = 0
metrics = PerformanceMetrics()
presenter = None
output_transform = None
input_crop = None
if args.crop_size[0] > 0 and args.crop_size[1] > 0:
input_crop = np.array(args.crop_size)
elif not (args.crop_size[0] == 0 and args.crop_size[1] == 0):
raise ValueError('Both crop height and width should be positive')
video_writer = cv2.VideoWriter()
while True:
start_time = perf_counter()
frame = cap.read()
if frame is None:
if frame_num == 0:
raise ValueError("Can't read an image from the input")
break
if input_crop:
frame = center_crop(frame, input_crop)
if frame_num == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
output_resolution):
raise RuntimeError("Can't open video writer")
detections = frame_processor.process(frame)
presenter.drawGraphs(frame)
frame = draw_detections(frame, frame_processor, detections,
output_transform)
metrics.update(start_time, frame)
frame_num += 1
if video_writer.isOpened() and (args.output_limit <= 0
or frame_num <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Face recognition demo', frame)
key = cv2.waitKey(1)
# Quit
if key in {ord('q'), ord('Q'), 27}:
break
presenter.handleKey(key)
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def main():
metrics = PerformanceMetrics()
args = build_argparser().parse_args()
log.info('Initializing Inference Engine...')
ie = IECore()
plugin_config = get_user_config(args.device, args.num_streams,
args.num_threads)
log.info('Loading network...')
model, visualizer = get_model(ie, args)
pipeline = AsyncPipeline(ie,
model,
plugin_config,
device=args.device,
max_num_requests=args.num_infer_requests)
cap = open_images_capture(args.input, args.loop)
next_frame_id = 0
next_frame_id_to_show = 0
log.info('Starting inference...')
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
presenter = None
output_transform = None
video_writer = cv2.VideoWriter()
only_masks = args.only_masks
while True:
if pipeline.is_ready():
# Get new image/frame
start_time = perf_counter()
frame = cap.read()
if frame is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
output_transform = OutputTransform(frame.shape[:2],
args.output_resolution)
if args.output_resolution:
output_resolution = output_transform.new_resolution
else:
output_resolution = (frame.shape[1], frame.shape[0])
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(output_resolution[0] / 4),
round(output_resolution[1] / 8)))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), output_resolution):
raise RuntimeError("Can't open video writer")
# Submit for inference
pipeline.submit_data(frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
# Wait for empty request
pipeline.await_any()
if pipeline.callback_exceptions:
raise pipeline.callback_exceptions[0]
# Process all completed requests
results = pipeline.get_result(next_frame_id_to_show)
if results:
objects, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
frame = render_segmentation(frame, objects, visualizer,
output_transform, only_masks)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
next_frame_id_to_show += 1
if not args.no_show:
cv2.imshow('Segmentation Results', frame)
key = cv2.waitKey(1)
if key == 27 or key == 'q' or key == 'Q':
break
if key == 9:
only_masks = not only_masks
presenter.handleKey(key)
pipeline.await_all()
# Process completed requests
for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
results = pipeline.get_result(next_frame_id_to_show)
while results is None:
results = pipeline.get_result(next_frame_id_to_show)
objects, frame_meta = results
frame = frame_meta['frame']
start_time = frame_meta['start_time']
frame = render_segmentation(frame, objects, visualizer,
output_transform, only_masks)
presenter.drawGraphs(frame)
metrics.update(start_time, frame)
if video_writer.isOpened() and (
args.output_limit <= 0
or next_frame_id_to_show <= args.output_limit - 1):
video_writer.write(frame)
if not args.no_show:
cv2.imshow('Segmentation Results', frame)
key = cv2.waitKey(1)
metrics.print_total()
print(presenter.reportMeans())
def main():
log.basicConfig(format='[ %(levelname)s ] %(message)s',
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
img_retrieval = ImageRetrieval(args.model, args.device, args.gallery,
INPUT_SIZE, args.cpu_extension)
cap = open_images_capture(args.input, args.loop)
if cap.get_type() not in ('VIDEO', 'CAMERA'):
raise RuntimeError(
"The input should be a video file or a numeric camera ID")
frames = RoiDetectorOnVideo(cap)
compute_embeddings_times = []
search_in_gallery_times = []
positions = []
frames_processed = 0
presenter = monitors.Presenter(args.utilization_monitors, 0)
video_writer = cv2.VideoWriter()
for image, view_frame in frames:
position = None
sorted_indexes = []
if image is not None:
image = central_crop(image, divide_by=5, shift=1)
elapsed, probe_embedding = time_elapsed(
img_retrieval.compute_embedding, image)
compute_embeddings_times.append(elapsed)
elapsed, (sorted_indexes, distances) = time_elapsed(
img_retrieval.search_in_gallery, probe_embedding)
search_in_gallery_times.append(elapsed)
sorted_classes = [
img_retrieval.gallery_classes[i] for i in sorted_indexes
]
if args.ground_truth is not None:
position = sorted_classes.index(
img_retrieval.text_label_to_class_id[args.ground_truth])
positions.append(position)
log.info("ROI detected, found: %d, position of target: %d",
sorted_classes[0], position)
else:
log.info("ROI detected, found: %s", sorted_classes[0])
image, key = visualize(
view_frame,
position, [img_retrieval.impaths[i] for i in sorted_indexes],
distances[sorted_indexes] if position is not None else None,
img_retrieval.input_size,
np.mean(compute_embeddings_times),
np.mean(search_in_gallery_times),
imshow_delay=3,
presenter=presenter,
no_show=args.no_show)
if frames_processed == 0:
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(image.shape[1], image.shape[0])):
raise RuntimeError("Can't open video writer")
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(image)
if key == 27:
break
print(presenter.reportMeans())
if positions:
compute_metrics(positions)
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
# Plugin initialization for specified device and load extensions library if specified
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
# Read IR
log.info('Reading Proposal model {}'.format(args.model_pnet))
p_net = core.read_model(args.model_pnet)
if len(p_net.inputs) != 1:
raise RuntimeError("Pnet supports only single input topologies")
if len(p_net.outputs) != 2:
raise RuntimeError("Pnet supports two output topologies")
log.info('Reading Refine model {}'.format(args.model_rnet))
r_net = core.read_model(args.model_rnet)
if len(r_net.inputs) != 1:
raise RuntimeError("Rnet supports only single input topologies")
if len(r_net.outputs) != 2:
raise RuntimeError("Rnet supports two output topologies")
log.info('Reading Output model {}'.format(args.model_onet))
o_net = core.read_model(args.model_onet)
if len(o_net.inputs) != 1:
raise RuntimeError("Onet supports only single input topologies")
if len(o_net.outputs) != 3:
raise RuntimeError("Onet supports three output topologies")
pnet_input_tensor_name = p_net.inputs[0].get_any_name()
rnet_input_tensor_name = r_net.inputs[0].get_any_name()
onet_input_tensor_name = o_net.inputs[0].get_any_name()
for node in p_net.outputs:
if node.shape[1] == 2:
pnet_cls_name = node.get_any_name()
elif node.shape[1] == 4:
pnet_roi_name = node.get_any_name()
else:
raise RuntimeError("Unsupported output layer for Pnet")
for node in r_net.outputs:
if node.shape[1] == 2:
rnet_cls_name = node.get_any_name()
elif node.shape[1] == 4:
rnet_roi_name = node.get_any_name()
else:
raise RuntimeError("Unsupported output layer for Rnet")
for node in o_net.outputs:
if node.shape[1] == 2:
onet_cls_name = node.get_any_name()
elif node.shape[1] == 4:
onet_roi_name = node.get_any_name()
elif node.shape[1] == 10:
onet_pts_name = node.get_any_name()
else:
raise RuntimeError("Unsupported output layer for Onet")
next_frame_id = 0
metrics = PerformanceMetrics()
presenter = None
video_writer = cv2.VideoWriter()
is_loaded_before = False
while True:
start_time = perf_counter()
origin_image = cap.read()
if origin_image is None:
if next_frame_id == 0:
raise ValueError("Can't read an image from the input")
break
if next_frame_id == 0:
presenter = monitors.Presenter(args.utilization_monitors, 55,
(round(origin_image.shape[1] / 4), round(origin_image.shape[0] / 8)))
if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
cap.fps(), (origin_image.shape[1], origin_image.shape[0])):
raise RuntimeError("Can't open video writer")
next_frame_id += 1
rgb_image = cv2.cvtColor(origin_image, cv2.COLOR_BGR2RGB)
oh, ow, _ = rgb_image.shape
scales = utils.calculate_scales(rgb_image)
# *************************************
# Pnet stage
# *************************************
pnet_res = []
for i, scale in enumerate(scales):
hs = int(oh*scale)
ws = int(ow*scale)
image = preprocess_image(rgb_image, ws, hs)
p_net.reshape({pnet_input_tensor_name: PartialShape([1, 3, ws, hs])}) # Change weidth and height of input blob
compiled_pnet = core.compile_model(p_net, args.device)
infer_request_pnet = compiled_pnet.create_infer_request()
if i == 0 and not is_loaded_before:
log.info("The Proposal model {} is loaded to {}".format(args.model_pnet, args.device))
infer_request_pnet.infer(inputs={pnet_input_tensor_name: image})
p_res = {name: infer_request_pnet.get_tensor(name).data[:] for name in {pnet_roi_name, pnet_cls_name}}
pnet_res.append(p_res)
image_num = len(scales)
rectangles = []
for i in range(image_num):
roi = pnet_res[i][pnet_roi_name]
cls = pnet_res[i][pnet_cls_name]
_, _, out_h, out_w = cls.shape
out_side = max(out_h, out_w)
rectangle = utils.detect_face_12net(cls[0][1], roi[0], out_side, 1/scales[i], ow, oh,
score_threshold[0], iou_threshold[0])
rectangles.extend(rectangle)
rectangles = utils.NMS(rectangles, iou_threshold[1], 'iou')
# Rnet stage
if len(rectangles) > 0:
r_net.reshape({rnet_input_tensor_name: PartialShape([len(rectangles), 3, 24, 24])}) # Change batch size of input blob
compiled_rnet = core.compile_model(r_net, args.device)
infer_request_rnet = compiled_rnet.create_infer_request()
if not is_loaded_before:
log.info("The Refine model {} is loaded to {}".format(args.model_rnet, args.device))
rnet_input = []
for rectangle in rectangles:
crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])]
crop_img = preprocess_image(crop_img, 24, 24)
rnet_input.extend(crop_img)
infer_request_rnet.infer(inputs={rnet_input_tensor_name: rnet_input})
rnet_res = {name: infer_request_rnet.get_tensor(name).data[:] for name in {rnet_roi_name, rnet_cls_name}}
roi = rnet_res[rnet_roi_name]
cls = rnet_res[rnet_cls_name]
rectangles = utils.filter_face_24net(cls, roi, rectangles, ow, oh, score_threshold[1], iou_threshold[2])
# Onet stage
if len(rectangles) > 0:
o_net.reshape({onet_input_tensor_name: PartialShape([len(rectangles), 3, 48, 48])}) # Change batch size of input blob
compiled_onet = core.compile_model(o_net, args.device)
infer_request_onet = compiled_onet.create_infer_request()
if not is_loaded_before:
log.info("The Output model {} is loaded to {}".format(args.model_onet, args.device))
is_loaded_before = True
onet_input = []
for rectangle in rectangles:
crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])]
crop_img = preprocess_image(crop_img, 48, 48)
onet_input.extend(crop_img)
infer_request_onet.infer(inputs={onet_input_tensor_name: onet_input})
onet_res = {name: infer_request_onet.get_tensor(name).data[:] for name in {onet_roi_name, onet_cls_name, onet_pts_name}}
roi = onet_res[onet_roi_name]
cls = onet_res[onet_cls_name]
pts = onet_res[onet_pts_name]
rectangles = utils.filter_face_48net(cls, roi, pts, rectangles, ow, oh,
score_threshold[2], iou_threshold[3])
# display results
for rectangle in rectangles:
# Draw detected boxes
cv2.putText(origin_image, 'confidence: {:.2f}'.format(rectangle[4]),
(int(rectangle[0]), int(rectangle[1])), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0))
cv2.rectangle(origin_image, (int(rectangle[0]), int(rectangle[1])), (int(rectangle[2]), int(rectangle[3])),
(255, 0, 0), 1)
# Draw landmarks
for i in range(5, 15, 2):
cv2.circle(origin_image, (int(rectangle[i+0]), int(rectangle[i+1])), 2, (0, 255, 0))
metrics.update(start_time, origin_image)
if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id <= args.output_limit):
video_writer.write(origin_image)
if not args.no_show:
cv2.imshow('MTCNN Results', origin_image)
key = cv2.waitKey(1)
if key in {ord('q'), ord('Q'), 27}:
break
presenter.handleKey(key)
metrics.log_total()
