def update(self, last_request_start_time, frame, position=(15, 30),
font_scale=0.75, color=(200, 10, 10), thickness=2):
current_time = perf_counter()
if self.last_update_time is None:
self.last_update_time = current_time
return
self.current_moving_statistic.latency += current_time - last_request_start_time
self.current_moving_statistic.period = current_time - self.last_update_time
self.current_moving_statistic.frame_count += 1
if current_time - self.last_update_time > self.time_window_size:
self.last_moving_statistic = self.current_moving_statistic
self.total_statistic.combine(self.last_moving_statistic)
self.current_moving_statistic = Statistic()
self.last_update_time = current_time
# Draw performance stats over frame
current_latency, current_fps = self.get_last()
if current_latency is not None:
put_highlighted_text(frame, "Latency: {:.1f} ms".format(current_latency * 1e3),
position, cv2.FONT_HERSHEY_COMPLEX, font_scale, color, thickness)
if current_fps is not None:
put_highlighted_text(frame, "FPS: {:.1f}".format(current_fps),
(position[0], position[1]+30), cv2.FONT_HERSHEY_COMPLEX, font_scale, color, thickness)
def move_car(pipeline, captured, rect):
arrived = False
# Create floor point tracker
rgb = cv2.cvtColor(captured, cv2.COLOR_BGR2RGB)
tracker = dlib.correlation_tracker()
rect = dlib.rectangle(rect[0], rect[1], rect[2], rect[3])
tracker.start_track(rgb, rect)
while not arrived:
frame = pipeline.wait_for_frames()
depth = frame.get_depth_frame()
frame = np.asanyarray(frame.get_color_frame().get_data())
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# Update the tracker
tracker.update(rgb)
pos = tracker.get_position()
# unpack the position object
startX = int(pos.left())
startY = int(pos.top())
endX = int(pos.right())
endY = int(pos.bottom())
put_highlighted_text(frame, "Status: traversing", (15, 20), cv2.FONT_HERSHEY_COMPLEX, 0.75, (200, 10, 10), 2)
cv2.rectangle(frame, (startX, startY), (endX, endY), (0, 0, 255), 2)
# Centroid pixel of distination
centroid = (int((startX + endX) / 2), int((startY + endY) / 2))
# Get average distance
dis_sum = 0
for i in range(10):
for j in range(10):
w = i - 5
h = j - 5
try:
dis_sum += depth.get_distance(centroid[0] + w, centroid[1] + h)
except:
dis_sum += 0
# Remain distance from destination
distance_btw_cam_and_object = dis_sum/100
# >>>>>>>>>>>>>>>>>>>>>>>>>>>> Move car <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
# print(centroid)
# print(distance_btw_cam_and_object)
# Show frame
frame = imutils.resize(frame, width=800)
cv2.imshow("Frame", frame)
key = cv2.waitKey(1) & 0xFF
ESC_KEY = 27
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
def main():
args = build_argparser().parse_args()
log.info('Initializing Inference Engine...')
ie = IECore()
config_user_specified, config_min_latency = get_plugin_configs(
args.device, args.num_streams, args.num_threads)
log.info('Loading network...')
completed_request_results = {}
modes = cycle(Modes)
prev_mode = mode = next(modes)
log.info('Using {} mode'.format(mode.name))
mode_info = {mode: ModeInfo()}
exceptions = []
if args.architecture_type == 'ae':
HPE = HPEAssociativeEmbedding
else:
HPE = HPEOpenPose
hpes = {
Modes.USER_SPECIFIED:
HPE(ie,
args.model,
target_size=args.tsize,
device=args.device,
plugin_config=config_user_specified,
results=completed_request_results,
max_num_requests=args.num_infer_requests,
caught_exceptions=exceptions),
Modes.MIN_LATENCY:
HPE(ie,
args.model,
target_size=args.tsize,
device=args.device.split(':')[-1].split(',')[0],
plugin_config=config_min_latency,
results=completed_request_results,
max_num_requests=1,
caught_exceptions=exceptions)
}
try:
input_stream = int(args.input)
except ValueError:
input_stream = args.input
cap = cv2.VideoCapture(input_stream)
wait_key_time = 1
next_frame_id = 0
next_frame_id_to_show = 0
input_repeats = 0
log.info('Starting inference...')
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
print(
"To switch between min_latency/user_specified modes, press TAB key in the output window"
)
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(cap.get(cv2.CAP_PROP_FRAME_WIDTH) / 4),
round(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) / 8)))
while (cap.isOpened() \
or completed_request_results \
or len(hpes[mode].empty_requests) < len(hpes[mode].requests)) \
and not exceptions:
if next_frame_id_to_show in completed_request_results:
frame_meta, raw_outputs = completed_request_results.pop(
next_frame_id_to_show)
poses, scores = hpes[mode].postprocess(raw_outputs, frame_meta)
valid_poses = scores > args.prob_threshold
poses = poses[valid_poses]
scores = scores[valid_poses]
frame = frame_meta['frame']
start_time = frame_meta['start_time']
if len(poses) and args.raw_output_message:
log.info('Poses:')
origin_im_size = frame.shape[:-1]
presenter.drawGraphs(frame)
show_poses(frame,
poses,
scores,
pose_score_threshold=args.prob_threshold,
point_score_threshold=args.prob_threshold)
if args.raw_output_message:
for pose, pose_score in zip(poses, scores):
pose_str = ' '.join(
'({:.2f}, {:.2f}, {:.2f})'.format(p[0], p[1], p[2])
for p in pose)
log.info('{} | {:.2f}'.format(pose_str, pose_score))
mode_message = '{} mode'.format(mode.name)
put_highlighted_text(frame, mode_message,
(10, int(origin_im_size[0] - 20)),
cv2.FONT_HERSHEY_COMPLEX, 0.75, (10, 10, 200),
2)
next_frame_id_to_show += 1
if prev_mode == mode:
mode_info[mode].frames_count += 1
elif len(completed_request_results) == 0:
mode_info[prev_mode].last_end_time = perf_counter()
prev_mode = mode
# Frames count is always zero if mode has just been switched (i.e. prev_mode != mode).
if mode_info[mode].frames_count != 0:
fps_message = 'FPS: {:.1f}'.format(mode_info[mode].frames_count / \
(perf_counter() - mode_info[mode].last_start_time))
mode_info[mode].latency_sum += perf_counter() - start_time
latency_message = 'Latency: {:.1f} ms'.format((mode_info[mode].latency_sum / \
mode_info[mode].frames_count) * 1e3)
# Draw performance stats over frame.
put_highlighted_text(frame, fps_message, (15, 20),
cv2.FONT_HERSHEY_COMPLEX, 0.75,
(200, 10, 10), 2)
put_highlighted_text(frame, latency_message, (15, 50),
cv2.FONT_HERSHEY_COMPLEX, 0.75,
(200, 10, 10), 2)
if not args.no_show:
cv2.imshow('Pose estimation results', frame)
key = cv2.waitKey(wait_key_time)
ESC_KEY = 27
TAB_KEY = 9
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
# Switch mode.
# Disable mode switch if the previous switch has not been finished yet.
if key == TAB_KEY and mode_info[mode].frames_count > 0:
mode = next(modes)
hpes[prev_mode].await_all()
mode_info[prev_mode].last_end_time = perf_counter()
mode_info[mode] = ModeInfo()
log.info('Using {} mode'.format(mode.name))
else:
presenter.handleKey(key)
elif hpes[mode].empty_requests and cap.isOpened():
start_time = perf_counter()
ret, frame = cap.read()
if not ret:
if input_repeats < args.loop or args.loop < 0:
cap.open(input_stream)
input_repeats += 1
else:
cap.release()
continue
hpes[mode](frame, next_frame_id, {
'frame': frame,
'start_time': start_time
})
next_frame_id += 1
else:
hpes[mode].await_any()
if exceptions:
raise exceptions[0]
for exec_net in hpes.values():
exec_net.await_all()
for mode_value, mode_stats in mode_info.items():
log.info('')
log.info('Mode: {}'.format(mode_value.name))
end_time = mode_stats.last_end_time if mode_stats.last_end_time is not None \
else perf_counter()
log.info('FPS: {:.1f}'.format(mode_stats.frames_count / \
(end_time - mode_stats.last_start_time)))
log.info('Latency: {:.1f} ms'.format((mode_stats.latency_sum / \
mode_stats.frames_count) * 1e3))
print(presenter.reportMeans())
def main():
args = build_argparser().parse_args()
# ------------- 1. Plugin initialization for specified device and load extensions library if specified -------------
log.info("Creating Inference Engine...")
ie = IECore()
config_user_specified = {}
config_min_latency = {}
devices_nstreams = {}
if args.num_streams:
devices_nstreams = {device: args.num_streams for device in ['CPU', 'GPU'] if device in args.device} \
if args.num_streams.isdigit() \
else dict([device.split(':') for device in args.num_streams.split(',')])
if 'CPU' in args.device:
if args.cpu_extension:
ie.add_extension(args.cpu_extension, 'CPU')
if args.number_threads is not None:
config_user_specified['CPU_THREADS_NUM'] = str(args.number_threads)
if 'CPU' in devices_nstreams:
config_user_specified['CPU_THROUGHPUT_STREAMS'] = devices_nstreams['CPU'] \
if int(devices_nstreams['CPU']) > 0 \
else 'CPU_THROUGHPUT_AUTO'
config_min_latency['CPU_THROUGHPUT_STREAMS'] = '1'
if 'GPU' in args.device:
if 'GPU' in devices_nstreams:
config_user_specified['GPU_THROUGHPUT_STREAMS'] = devices_nstreams['GPU'] \
if int(devices_nstreams['GPU']) > 0 \
else 'GPU_THROUGHPUT_AUTO'
config_min_latency['GPU_THROUGHPUT_STREAMS'] = '1'
# -------------------- 2. Reading the IR generated by the Model Optimizer (.xml and .bin files) --------------------
log.info("Loading network")
net = ie.read_network(args.model, os.path.splitext(args.model)[0] + ".bin")
output_info = get_output_info(net)
assert len(
net.input_info
) == 1, "Sample supports only YOLO V3 based single input topologies"
# ---------------------------------------------- 3. Preparing inputs -----------------------------------------------
log.info("Preparing inputs")
input_blob = next(iter(net.input_info))
# Read and pre-process input images
if net.input_info[input_blob].input_data.shape[1] == 3:
input_height, input_width = net.input_info[
input_blob].input_data.shape[2:]
nchw_shape = True
else:
input_height, input_width = net.input_info[
input_blob].input_data.shape[1:3]
nchw_shape = False
if args.labels:
with open(args.labels, 'r') as f:
labels_map = [x.strip() for x in f]
else:
labels_map = None
input_stream = 0 if args.input == "cam" else args.input
mode = Mode(Modes.USER_SPECIFIED)
cap = cv2.VideoCapture(input_stream)
wait_key_time = 1
# ----------------------------------------- 4. Loading model to the plugin -----------------------------------------
log.info("Loading model to the plugin")
exec_nets = {}
exec_nets[Modes.USER_SPECIFIED] = ie.load_network(
network=net,
device_name=args.device,
config=config_user_specified,
num_requests=args.num_infer_requests)
exec_nets[Modes.MIN_LATENCY] = ie.load_network(
network=net,
device_name=args.device.split(":")[-1].split(",")[0],
config=config_min_latency,
num_requests=1)
empty_requests = deque(exec_nets[mode.current].requests)
completed_request_results = {}
next_frame_id = 0
next_frame_id_to_show = 0
mode_metrics = {mode.current: PerformanceMetrics()}
prev_mode_active_request_count = 0
event = threading.Event()
callback_exceptions = []
# ----------------------------------------------- 5. Doing inference -----------------------------------------------
log.info("Starting inference...")
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
print(
"To switch between min_latency/user_specified modes, press TAB key in the output window"
)
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(cap.get(cv2.CAP_PROP_FRAME_WIDTH) / 4),
round(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) / 8)))
while (cap.isOpened() \
or completed_request_results \
or len(empty_requests) < len(exec_nets[mode.current].requests)) \
and not callback_exceptions:
if next_frame_id_to_show in completed_request_results:
frame, output, start_time, is_same_mode = completed_request_results.pop(
next_frame_id_to_show)
next_frame_id_to_show += 1
objects = get_objects(output, output_info,
(input_height, input_width),
frame.shape[:-1], args.prob_threshold,
args.keep_aspect_ratio)
objects = filter_objects(objects, args.iou_threshold,
args.prob_threshold)
if len(objects) and args.raw_output_message:
log.info(
" Class ID | Confidence | XMIN | YMIN | XMAX | YMAX | COLOR "
)
origin_im_size = frame.shape[:-1]
presenter.drawGraphs(frame)
for obj in objects:
# Validation bbox of detected object
obj['xmax'] = min(obj['xmax'], origin_im_size[1])
obj['ymax'] = min(obj['ymax'], origin_im_size[0])
obj['xmin'] = max(obj['xmin'], 0)
obj['ymin'] = max(obj['ymin'], 0)
color = (min(obj['class_id'] * 12.5,
255), min(obj['class_id'] * 7,
255), min(obj['class_id'] * 5, 255))
det_label = labels_map[obj['class_id']] if labels_map and len(labels_map) >= obj['class_id'] else \
str(obj['class_id'])
if args.raw_output_message:
log.info(
"{:^9} | {:10f} | {:4} | {:4} | {:4} | {:4} | {} ".
format(det_label, obj['confidence'], obj['xmin'],
obj['ymin'], obj['xmax'], obj['ymax'], color))
cv2.rectangle(frame, (obj['xmin'], obj['ymin']),
(obj['xmax'], obj['ymax']), color, 2)
cv2.putText(
frame, "#" + det_label + ' ' +
str(round(obj['confidence'] * 100, 1)) + ' %',
(obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX,
0.6, color, 1)
helpers.put_highlighted_text(frame,
"{} mode".format(mode.current.name),
(10, int(origin_im_size[0] - 20)),
cv2.FONT_HERSHEY_COMPLEX, 0.75,
(10, 10, 200), 2)
if is_same_mode and prev_mode_active_request_count == 0:
mode_metrics[mode.current].update(start_time, frame)
else:
mode_metrics[mode.get_other()].update(start_time, frame)
prev_mode_active_request_count -= 1
helpers.put_highlighted_text(
frame, "Switching modes, please wait...",
(10, int(origin_im_size[0] - 50)),
cv2.FONT_HERSHEY_COMPLEX, 0.75, (10, 200, 10), 2)
if not args.no_show:
cv2.imshow("Detection Results", frame)
key = cv2.waitKey(wait_key_time)
if key in {ord("q"), ord("Q"), 27}: # ESC key
break
if key == 9: # Tab key
if prev_mode_active_request_count == 0:
prev_mode = mode.current
mode.switch()
prev_mode_active_request_count = len(
exec_nets[prev_mode].requests) - len(
empty_requests)
empty_requests.clear()
empty_requests.extend(exec_nets[mode.current].requests)
mode_metrics[mode.current] = PerformanceMetrics()
else:
presenter.handleKey(key)
elif empty_requests and prev_mode_active_request_count == 0 and cap.isOpened(
):
start_time = perf_counter()
ret, frame = cap.read()
if not ret:
if args.loop:
cap.open(input_stream)
else:
cap.release()
continue
request = empty_requests.popleft()
# resize input_frame to network size
in_frame = preprocess_frame(frame, input_height, input_width,
nchw_shape, args.keep_aspect_ratio)
# Start inference
request.set_completion_callback(
py_callback=async_callback,
py_data=(request, next_frame_id, mode.current, frame,
start_time, completed_request_results, empty_requests,
mode, event, callback_exceptions))
request.async_infer(inputs={input_blob: in_frame})
next_frame_id += 1
else:
event.wait()
event.clear()
if callback_exceptions:
raise callback_exceptions[0]
for mode, metrics in mode_metrics.items():
print("\nMode: {}".format(mode.name))
metrics.print_total()
print(presenter.reportMeans())
for exec_net in exec_nets.values():
await_requests_completion(exec_net.requests)
def main():
modeSwitched = False
args = build_argparser().parse_args()
log.info('Initializing Inference Engine...')
ie = IECore()
log.info('Loading network...')
completed_request_results = {}
modes = cycle(Modes)
prev_mode = mode = next(modes)
log.info('Using {} mode'.format(mode.name))
mode_info = {mode: ModeInfo()}
exceptions = []
if args.architecture_type == 'ae':
HPE = HPEAssociativeEmbedding
else:
HPE = HPEOpenPose
hpes = {
Modes.USER_SPECIFIED:
HPE(ie, args.model, target_size=args.tsize, device=args.device, plugin_config={},
results=completed_request_results, max_num_requests=args.num_infer_requests,
caught_exceptions=exceptions),
Modes.MIN_LATENCY:
HPE(ie, args.model, target_size=args.tsize, device=args.device.split(':')[-1].split(',')[0],
plugin_config={}, results=completed_request_results, max_num_requests=1,
caught_exceptions=exceptions)
}
# grab a reference to the webcam
print("[INFO] starting video stream...")
pipeline = rs.pipeline()
pipeConfig = rs.config()
pipeConfig.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)
pipeConfig.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
pipeline.start(pipeConfig)
time.sleep(2.0)
wait_key_time = 1
next_frame_id = 0
next_frame_id_to_show = 0
input_repeats = 0
log.info('Starting inference...')
print("To close the application, press 'CTRL+C' here or switch to the output window and press ESC key")
print("To switch between min_latency/user_specified modes, press TAB key in the output window")
while not exceptions:
if next_frame_id_to_show in completed_request_results:
frame_meta, raw_outputs = completed_request_results.pop(next_frame_id_to_show)
poses, scores = hpes[mode].postprocess(raw_outputs, frame_meta)
valid_poses = scores > args.prob_threshold
poses = poses[valid_poses]
scores = scores[valid_poses]
frame = frame_meta['frame']
start_time = frame_meta['start_time']
# Detect cough
idx, rect = detect_cough(poses, scores)
if idx != -1:
move_car(pipeline, frame, rect)
origin_im_size = frame.shape[:-1]
show_poses(frame, poses, scores, pose_score_threshold=args.prob_threshold,
point_score_threshold=args.prob_threshold)
next_frame_id_to_show += 1
if prev_mode == mode:
mode_info[mode].frames_count += 1
elif len(completed_request_results) == 0:
mode_info[prev_mode].last_end_time = perf_counter()
prev_mode = mode
# Frames count is always zero if mode has just been switched (i.e. prev_mode != mode).
if mode_info[mode].frames_count != 0:
fps_message = 'FPS: {:.1f}'.format(mode_info[mode].frames_count / \
(perf_counter() - mode_info[mode].last_start_time))
mode_info[mode].latency_sum += perf_counter() - start_time
latency_message = 'Latency: {:.1f} ms'.format((mode_info[mode].latency_sum / \
mode_info[mode].frames_count) * 1e3)
# Draw performance stats over frame.
put_highlighted_text(frame, fps_message, (15, 20), cv2.FONT_HERSHEY_COMPLEX, 0.75, (200, 10, 10), 2)
put_highlighted_text(frame, latency_message, (15, 50), cv2.FONT_HERSHEY_COMPLEX, 0.75, (200, 10, 10), 2)
if not args.no_show:
frame = imutils.resize(frame, width=800)
cv2.imshow("Frame", frame)
key = cv2.waitKey(1) & 0xFF
#cv2.imwrite('output.png', frame)
# key = cv2.waitKey(wait_key_time)
key = 0
ESC_KEY = 27
TAB_KEY = 9
# Quit.
if key in {ord('q'), ord('Q'), ESC_KEY}:
break
# Switch mode.
# Disable mode switch if the previous switch has not been finished yet.
# if key == TAB_KEY and mode_info[mode].frames_count > 0:
if not modeSwitched and mode_info[mode].frames_count > 0:
modeSwitched = True
mode = next(modes)
hpes[prev_mode].await_all()
mode_info[prev_mode].last_end_time = perf_counter()
mode_info[mode] = ModeInfo()
log.info('Using {} mode'.format(mode.name))
elif hpes[mode].empty_requests:
start_time = perf_counter()
# grab the next frame and handle
frame = pipeline.wait_for_frames()
depth = frame.get_depth_frame()
frame = np.asanyarray(frame.get_color_frame().get_data())
hpes[mode](frame, next_frame_id, {'frame': frame, 'start_time': start_time})
next_frame_id += 1
else:
hpes[mode].await_any()
if exceptions:
raise exceptions[0]
for exec_net in hpes.values():
exec_net.await_all()