def __shapes(self):
for id, shape in shapetracker().shapes.items():
{
'fresh': self.__fresh_shape,
'visible': self.__visible_shape,
'covered': self.__covered_shape,
}[shape.state](id, shape)
if shape.pressed:
self.__pressed_shape(shape)
for kps in shape.keypoints.values():
for kp in kps:
cv2.drawMarker(self.frame, tuple(kp.astype(int)), (0, 255, 0))
for shape in shapetracker().pending_shapes:
self.__pending_shape(shape)
def next_frame(self):
self.frame += 1
self.publish('frame_begins', {'frame': self.frame})
if not realsensecam().acquire_frames():
return None
handdetector().determine_hand_cnt()
if handtracker().update() and not ui().menu_active:
detected_shapes = shapedetector().detect_shapes()
shapetracker().process_detected_shapes(detected_shapes)
touchedshapetracker().update()
if self.logger is not None:
self.logger.logAll()
return visualizer().visualize()
def logAll(self):
l = {'fingertip': {'x': -1, 'y': -1}, 'fingerdown': False, 'shape': []}
if handdetector().hand_cnt is not None and handdetector().hand_valid:
l['fingertip'] = {
'x': int(handdetector().fingertip_pos[0]),
'y': int(handdetector().fingertip_pos[1])
}
if handtracker().finger_down:
l['fingerdown'] = True
if len(shapetracker().shapes) > 0:
cnt = next(iter(shapetracker().shapes.values())).cnt
eps = .1 * cv2.arcLength(cnt, True)
approxPts = cv2.approxPolyDP(cnt, eps, True)[:, 0]
for pos in approxPts:
l['shape'].append({'x': int(pos[0]), 'y': int(pos[1])})
self.log.append(l)
def __lost_shapes(self):
for idx, lost_shape in enumerate(shapetracker().lost_shapes.values()):
fac = self.lost_shapes_h / max(lost_shape.footprint.shape)
scaled_footprint = cv2.resize(lost_shape.footprint, (0, 0), fx=fac, fy=fac)
nz_idx = list(np.nonzero(scaled_footprint))
nz_idx[0] += self.lost_shapes_y
nz_idx[1] += idx * (self.lost_shapes_h + 2)
self.frame[tuple(nz_idx)] = (255, 255, 255)
def update(self):
touched_shape = handtracker().touched_shape
if touched_shape is None:
return
# Create a mask that should only contain the shape that is being moved, without the finger
new_hand_mask = handdetector().most_recent_mask
shape_mask = shapedetector().most_recent_mask
new_mask_to_search = cv2.bitwise_and(shape_mask,
shape_mask,
mask=(255 - new_hand_mask))
for shape in shapetracker().shapes.values():
if shape != touched_shape:
cv2.drawContours(new_mask_to_search, [shape.cnt], 0, 0, -1)
cv2.drawContours(new_mask_to_search, [shape.cnt], 0, 0, 5)
new_mask_to_search = cv2.morphologyEx(new_mask_to_search,
cv2.MORPH_OPEN,
np.ones((5, 5), np.uint8))
if self.old_mask_to_search is None:
self.old_mask_to_search = new_mask_to_search
self.initial_mask_to_search = new_mask_to_search
self.old_hand_mask = new_hand_mask
return # The prediction will only be possible in the next frame
# Calculate xor between last shape footprint (since it has last moved, ignoring swipes) and its current footprint
xored_mask = cv2.bitwise_xor(self.old_mask_to_search,
new_mask_to_search)
xored_mask = cv2.bitwise_and(xored_mask, 255 - self.old_hand_mask)
xored_mask = cv2.bitwise_and(xored_mask, 255 - new_hand_mask)
xored_mask = cv2.morphologyEx(xored_mask, cv2.MORPH_OPEN,
np.ones((2, 2)))
# Use this metric to determine if the shape has moved significantly
amount_moved_pixels = np.count_nonzero(xored_mask)
if amount_moved_pixels > 200:
self.move = True
if not touched_shape.moving:
touched_shape.start_moving(
np.array(handdetector().fingertip_pos))
# Calculate and apply transform
angle = touched_shape.transform_to_fit_masks(
self.initial_mask_to_search, new_mask_to_search,
-handtracker().finger_deg_delta)[1]
# Prepare next iteration
self.old_mask_to_search = new_mask_to_search
self.old_hand_mask = new_hand_mask
else:
self.move = False
touched_shape.stop_moving()
def __init__(self, logger=None):
super().__init__()
if conf()['compile_pyx_on_startup']:
os.system("python setup.py build_ext --inplace")
self.frame = 0
self.logger = logger
# Initialize all needed modules
shapedetector()
shapetracker()
touchedshapetracker()
handdetector()
handtracker()
visualizer()
ui()
shapepicker()
shaperegionpicker()
# Put the wires together
handtracker().subscribe('hand_exit', shapetracker().clear_lost_shapes)
handtracker().subscribe('finger_up',
touchedshapetracker().on_finger_up)
handtracker().subscribe('finger_pressing', ui().on_finger_pressing)
handtracker().subscribe('finger_up', ui().on_finger_up)
handtracker().subscribe('finger_down', ui().on_finger_down)
handtracker().subscribe('finger_pressing',
shapepicker().on_finger_pressing)
handtracker().subscribe('finger_up', shapepicker().on_finger_up)
handtracker().subscribe('hand_exit', handdetector().on_hand_exit)
handtracker().subscribe('finger_pressing',
shaperegionpicker().on_finger_pressing)
handtracker().subscribe('finger_up', shaperegionpicker().on_finger_up)
handtracker().subscribe('hand_exit', shaperegionpicker().on_hand_exit)
handtracker().subscribe('hand_exit',
shapepositionpicker().on_hand_exit)
def __init__(self, cnt):
super().__init__()
# Calculate the bbox
bbox = Bbox(*cv2.boundingRect(cnt))
# Draw an isolated footprint of the shape
offset = tuple(- np.array(bbox.position()))
isolated = np.zeros(bbox.size(True), np.uint8)
cv2.drawContours(isolated, [cnt], 0, 255, -1, offset=offset)
footprint = cv2.copyMakeBorder(isolated, 15, 15, 15, 15, cv2.BORDER_CONSTANT, 0)
# Determine the color of the shape
x, y, w, h = bbox.xywh()
patch = realsensecam().bgr[y:y + h, x:x + w, :][int(h / 3):int(2 * h / 3), int(w / 3):int(2 * w / 3), :]
patch = cv2.GaussianBlur(patch, (51, 51), 0)
if patch is None:
color = (0, 0, 0)
else:
ph, pw, _ = patch.shape
color = patch[int(ph / 2), int(pw / 2)]
color = tuple([int(x) for x in color])
color_hsv = cv2.cvtColor(np.array([[color]], np.uint8), cv2.COLOR_BGR2HSV)[0][0]
self.cnt = cnt
self.bbox = bbox
self.color = color
self.color_hsv = color_hsv
self.footprint = footprint
self.angle = 0
self.state = 'fresh'
self.state_stable_since = shapetracker().epoch
self.pressed = False
self.moving = False
self.initial_swipe_xy = None
self.current_swipe_xy = None
self.initial_move_xy = None
self.current_move_xy = None
self.initial_degs = 0
self.current_degs = None
self.cnt_on_down = None
self.needs_transform_to_fit_shape = False
self.keypoints = {}
self.keypoints_on_down = None
self.action_name = ""
def on_finger_pressing(self, _, data):
if not self.active or self.must_lift_finger:
return
xy = data['fingertip_pos']
found = False
for id, shape in shapetracker().shapes.items():
if shape.bbox.contains(*xy):
if shape != self.shape:
self.shape = shape
self.stable_since = time.time()
found = True
break
if not found:
self.shape = None
self.stable_since = time.time()
if time.time() - self.stable_since >= 1:
self.active = False
self.shape = None
self.callback(shape,
*self.args) # Picking completed, call the callback
return
def __stats(self):
self.nth_frame += 1
elapsed = time.time() - self.start_time
if elapsed > .1:
self.fps = int(self.nth_frame / elapsed)
self.nth_frame = 0
self.start_time = time.time()
self.__print("Tracking {:2d} shapes".format(len(shapetracker().shapes)), 2, self.text_l)
self.__print("{:2d} FPS".format(self.fps), 2, self.text_r - 86)
if not handdetector().hand_valid:
self.__print("HAND ERROR! Too many or too long hands.", 1, self.text_l, color=(0, 0, 255))
elif ui().menu_armed:
self.__print("Hold the blue square to enter the menu.", 1, self.text_l, color=(255, 50, 50))
elif ui().menu_active:
self.__print("Pick an option from the menu.", 1, self.text_l, color=(255, 50, 50))
elif ui().action_menu_armed:
self.__print("Hold the blue square to enter the menu.", 1, self.text_l, color=(50, 255, 50))
elif ui().action_menu_active:
self.__print("Pick an option from the menu.", 1, self.text_l, color=(50, 255, 50))
elif shapepicker().active:
if shapepicker().must_lift_finger:
self.__print("Please lift your finger.", 0, self.text_l, color=(0, 255, 255))
else:
self.__print(shapepicker().hint, 0, self.text_l, color=(0, 255, 0))
self.__print("Press and hold the shape you want to pick ({}%)".format(int(100 * shapepicker().progress())), 1, self.text_l, color=(0, 255, 0))
elif shaperegionpicker().active:
if shaperegionpicker().must_lift_finger:
self.__print("Please lift your finger (keep hand under camera).", 0, self.text_l, color=(0, 255, 255))
else:
self.__print("Press and hold {} in the shape ({}%)".format(shaperegionpicker().current_region_description(), int(100 * shaperegionpicker().progress())), 0, self.text_l, color=(0, 255, 0))
self.__print("When all regions are selected, remove your hand.", 1, self.text_l, color=(0, 255, 0))
elif shapepositionpicker().active:
self.__print(shapepositionpicker().hint, 0, self.text_l, color=(0, 255, 0))
self.__print("When the position has been reached, remove hand.", 1, self.text_l, color=(0, 255, 0))
def __fresh_shape(self, id, shape):
x, y = shape.bbox.center()
cv2.rectangle(self.frame, (x - 15, y - 4), (x - 15 + int(shapetracker().percentage_for_shape(shape) * 30), y + 4), (0, 200, 0), cv2.FILLED)
cv2.rectangle(self.frame, (x - 15, y - 4), (x + 15, y + 4), (200, 200, 200))
cv2.drawContours(self.frame, [shape.cnt], 0, (0, 255, 0), 1)
def set_state(self, new_state):
self.state = new_state
self.state_stable_since = shapetracker().epoch
def update(self):
if not handdetector().hand_valid:
return False
# Check if the visibility of the hand has changed
now_visible = handdetector().hand_cnt is not None
was_visible = self.hand_visible
self.hand_visible = now_visible
if now_visible and not was_visible:
self.publish('hand_enter', None)
if not now_visible and was_visible:
self.publish('hand_exit', None)
# Check if finger is down or not
now_down = handdetector().fingertip_height < conf(
)['finger_height_threshold']
was_down = self.finger_down
self.finger_down = now_down
if now_down and not was_down:
self.__start_of_tracking()
self.__update_of()
data = {
'fingertip_pos': self.enhanced_fingertip_pos(),
'finger_delta': self.finger_delta,
'finger_deg_delta': self.finger_deg_delta
}
self.publish('finger_down', data)
touchetmanager().emit_global_event('finger_down', data)
for shape in shapetracker().shapes.values():
shape.on_finger_down(
data) # The shape will figure out if it is the target
if not now_down and was_down:
self.__stop_of_tracking()
data = {
'fingertip_pos': self.enhanced_fingertip_pos(),
'finger_delta': self.finger_delta,
'finger_deg_delta': self.finger_deg_delta
}
self.publish('finger_up', data)
touchetmanager().emit_global_event('finger_up', data)
for shape in shapetracker().shapes.values():
shape.on_finger_up(
data) # The shape will figure out if it is the target
if was_down and now_down:
data = {
'fingertip_pos': self.enhanced_fingertip_pos(),
'finger_delta': self.finger_delta,
'finger_deg_delta': self.finger_deg_delta
}
self.publish('finger_pressing', data)
touchetmanager().emit_global_event('finger_pressing', data)
for shape in shapetracker().shapes.values():
shape.on_finger_pressing(
data) # The shape will figure out if it is the target
if self.__update_of(
): # Only trigger if the finger actually moved enough
self.publish('finger_moved', data)
touchetmanager().emit_global_event('finger_moved', data)
for shape in shapetracker().shapes.values():
shape.on_finger_moved(
data) # The shape will figure out if it is the target
return True