def draw_annotations(self, resized_frame):
frame = resized_frame
point = self.annotations[0]['point']
hand = self.annotations[0]['hand']
frame = crosshead(frame, point)
frame = text(frame, hand, point=point)
return crosshead(frame, self.annotations[0]['point'])
def run(self):
capture = cv2.VideoCapture(self.camera)
capture.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
last_time = time.time()
fps = None
while True:
ret, frame = capture.read()
if not ret:
break
result = self.inference_session.cv2_run(frame)
now = time.time()
fps = accumulate(fps, 1 / (now - last_time))
last_time = now
frame = draw_inferred_crossheads(frame, result)
show_inference_result(frame, result)
cv2.imshow(
'Camera',
text(cv2.flip(frame, 1), f"{fps:.1f} fps | Press ESC to quit"))
if cv2.waitKey(1) & 0xFF == 27: # esc to quit
break
capture.release()
cv2.destroyAllWindows()
def handle_frame(self, at, frame):
display = cv2.vconcat([
text(
np.zeros_like(frame)[:50],
f'{self.gesture_name}: {self.capturing_session.message(at)}'
if self.capturing_session else
'ESC: quit; a: start/stop; d: delete',
),
self.capturing_session.process(at, frame)
if self.capturing_session else cv2.flip(frame, 1),
self.playback_session.render(at, frame.shape[:-1][::-1])
if self.playback_session else text(
np.zeros_like(frame),
'Last labeled image will be here',
),
])
cv2.imshow('Capture and Label', display)
def draw_annotations(self, filipped_frame):
frame = filipped_frame
point = self.annotations[0]['point']
hand = self.annotations[0]['hand']
flipped_point = (1 - point[0], point[1])
frame = crosshead(frame, flipped_point)
frame = text(frame, hand, point=flipped_point)
return frame
def run(self):
fps = None
latency = None
with self.pipeline.threaded() as stream:
for item in stream:
fps = accumulate(fps, item.fps)
latency = accumulate(latency, item.latency)
frame = item.frame
frame = draw_inferred_crossheads(frame, item.inference_result)
frame = cv2.flip(frame, 1)
frame = text(frame, f"fps {fps: 2.0f}", point=(0, .5))
frame = text(frame, f"latency {latency:.2f}s", point=(0, .75))
frame = text(frame, "Press ESC to quit")
cv2.imshow('Camera', frame)
show_inference_result(frame, item.inference_result)
if cv2.waitKey(1) & 0xFF == 27: # esc to quit
break
cv2.destroyAllWindows()
def draw_annotations(self, resized_frame):
frame = resized_frame
for annotation in self.annotations:
x = annotation['x']
y = annotation['y']
frame = crosshead(frame, x, y)
if annotation['label'] == 'open_pinch_top':
hand = annotation['hand']
frame = text(frame, hand, point=(x, y))
return frame
def draw_annotations(self, filipped_frame):
frame = filipped_frame
for annotation in self.annotations:
x = annotation['x']
y = annotation['y']
flipped_point = (1 - x, y)
frame = crosshead(frame, *flipped_point)
if annotation['label'] == 'open_pinch_top':
hand = annotation['hand']
frame = text(frame, hand, point=flipped_point)
return frame
def run(self):
scrolling_thread = threading.Thread(target=self.scroll_forever, daemon=True)
scrolling_thread.start()
button_down = None
button_down_since = None
last_click = None
with self.pipeline.threaded() as stream:
for item in stream:
now = item.captured_at
frame = item.frame
inference_result = item.inference_result
(left, open_left), (right, open_right) = inference_result
# actions
left_x, left_y = relative_average_coordinate(left, (1, 0))
right_x, right_y = relative_average_coordinate(right, (1, 0))
button_down_now = None
scroll_now = 0
if left.max() < self.score_threshold or right.max() < self.score_threshold:
pass
elif open_left[left > .5].mean() > .5 and open_right[right > .5].mean() > .5:
button_down_now = 'drag'
elif open_left[left > .5].mean() > .5 or open_right[right > .5].mean() > .5:
pass
elif left_x < right_x:
button_down_now = 'double click'
elif abs(left_y - right_y) < .05:
if left_x - right_x < .1:
button_down_now = 'click'
else:
button_down_now = 'right click'
elif abs(left_y - right_y) > .08:
scroll_now = left_y - right_y
if button_down != button_down_now:
button_down_since = now
if button_down == 'drag':
self.mouse.release(pynput.mouse.Button.left)
if button_down_now == 'drag' and (last_click is None or now - last_click > .5):
self.mouse.press(pynput.mouse.Button.left)
last_click = now
if button_down_now == 'click' and (last_click is None or now - last_click > .5):
self.mouse.click(pynput.mouse.Button.left)
last_click = now
if button_down_now == 'double click' and (last_click is None or now - last_click > .5):
self.mouse.click(pynput.mouse.Button.left, 2)
last_click = now
button_down = button_down_now
elif button_down_now == 'right click' and .5 < now - button_down_since and \
(last_click is None or last_click < button_down_since):
self.mouse.click(pynput.mouse.Button.right)
last_click = now
self.scrolling_speed = scroll_now
frame = draw_inferred_crossheads(frame, inference_result)
cv2.imshow('Camera', text(cv2.flip(frame, 1), "Press ESC to quit"))
show_inference_result(frame, inference_result)
if cv2.waitKey(1) & 0xFF == 27: # esc to quit
break
cv2.destroyAllWindows()
def run(self):
scrolling_thread = threading.Thread(target=self.scroll_forever)
scrolling_thread.daemon = True
scrolling_thread.start()
capture = cv2.VideoCapture(self.camera)
capture.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
button_down = None
button_down_since = None
last_click = None
while True:
ret, frame = capture.read()
if not ret:
break
now = time.time()
# inference
inference_result = self.inference_session.cv2_run(frame)
left, right = inference_result
# actions
left_x, left_y = relative_average_coordinate(left, (1, 0))
right_x, right_y = relative_average_coordinate(right, (1, 0))
button_down_now = None
scroll_now = 0
if left.max() < self.score_threshold or right.max(
) < self.score_threshold:
pass
elif left_x < right_x:
button_down_now = 'double click'
elif abs(left_y - right_y) < .05:
if left_x - right_x < .1:
button_down_now = 'click'
else:
button_down_now = 'right click'
elif abs(left_y - right_y) > .1:
scroll_now = left_y - right_y
if button_down != button_down_now:
button_down_since = now
if button_down_now == 'click' and (last_click is None
or now - last_click > .5):
self.mouse.click(pynput.mouse.Button.left)
last_click = now
if button_down_now == 'double click' and (
last_click is None or now - last_click > .5):
self.mouse.click(pynput.mouse.Button.left, 2)
last_click = now
button_down = button_down_now
elif button_down_now == 'right click' and .5 < now - button_down_since and \
(last_click is None or last_click < button_down_since):
self.mouse.click(pynput.mouse.Button.right)
last_click = now
self.scrolling_speed = scroll_now
frame = draw_inferred_crossheads(frame, inference_result)
cv2.imshow('Camera', text(cv2.flip(frame, 1), "Press ESC to quit"))
show_inference_result(frame, inference_result)
if cv2.waitKey(1) & 0xFF == 27: # esc to quit
break
capture.release()
cv2.destroyAllWindows()