def __get_slice_img(self, lowest_layer, highest_layer):
marker_points = []
for i in range(lowest_layer, highest_layer):
largest_cnt = self.__get_layer_cnt(i)
if largest_cnt is not None:
hull = cv2.convexHull(largest_cnt, returnPoints=False)
defects = cv2.convexityDefects(largest_cnt, hull)
if defects is None:
continue
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(largest_cnt[s][0])
end = tuple(largest_cnt[e][0])
far = tuple(largest_cnt[f][0])
marker_points.append(start)
marker_points.append(end)
marker_img = np.zeros((realsensecam().H, realsensecam().W), np.uint8)
for op in marker_points:
cv2.circle(marker_img, op, 1, 255, -1)
marker_img = cv2.morphologyEx(
marker_img, cv2.MORPH_CLOSE,
cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (15, 15)))
return marker_img
def transform_to_fit_shape(self, other_shape):
my_mask = np.zeros((realsensecam().H, realsensecam().W), np.uint8)
other_mask = np.zeros_like(my_mask)
cv2.drawContours(my_mask, [self.cnt], 0, 255, -1)
cv2.drawContours(other_mask, [other_shape.cnt], 0, 255, -1)
translation, angle = self.transform_to_fit_masks(my_mask, other_mask, origin_is_on_down=False)
if abs(angle) < 8: # Otherwise we will need to do this again
self.needs_transform_to_fit_shape = False
self.publish('transformation_adjusted', {'shape': self, 'shape_move_delta': translation, 'shape_degs': -angle})
def __init__(self, bg_color):
self.bg_color = bg_color
self.items = {}
self.most_recent_mc = None
self.mc_stable_since = 0
self.margin = 20
self.outter_w = realsensecam().W / 3
self.inner_w = self.outter_w - 2 * self.margin
self.outter_h = realsensecam().H / 6
self.inner_h = self.outter_h - 2 * self.margin
def __init__(self):
self.lost_shapes_y = realsensecam().H + 2
self.lost_shapes_h = 30
self.text_h = 40
self.text_l = 0
self.text_r = realsensecam().W
self.text_y0 = self.lost_shapes_y + self.lost_shapes_h + 2
self.text_y = [self.text_y0, self.text_y0 + self.text_h - 23, self.text_y0 + self.text_h - 4]
self.text_size = 0.9
self.start_time = time.time()
self.nth_frame = 0
self.fps = 0
def on_finger_down(self, _, data):
xy = data['fingertip_pos']
if xy[1] < 50:
if xy[0] < 50:
self.menu_armed = True
self.finger_down_ts = time.time()
elif xy[0] > realsensecam().W - 50:
self.action_menu_armed = True
self.finger_down_ts = time.time()
def __get_layer_cnt(self, layer):
_, depth_th = cv2.threshold(realsensecam().depth_blurred, layer, 255,
cv2.THRESH_BINARY)
contours, _ = cv2.findContours(depth_th, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
if len(contours) > 0:
return max(contours, key=lambda c: cv2.contourArea(c))
else:
return None
def cnt_intersects_with_hand(self, cnt, include_secondary_contours=True):
if self.hand_cnt is None:
return False
# Create two empty frames, draw the contours and perform bitwise and
hand_img = np.zeros((realsensecam().H, realsensecam().W, 1), np.uint8)
cnt_img = hand_img.copy()
contours_to_check = [self.hand_cnt]
if include_secondary_contours:
contours_to_check += self.secondary_hand_cnts
cv2.drawContours(hand_img, contours_to_check, -1, 255, -1)
cv2.drawContours(hand_img, contours_to_check, -1, 255,
conf()['hand_shape_intersection_border'])
cv2.drawContours(cnt_img, [cnt], 0, 255, cv2.FILLED)
anded = np.bitwise_and(hand_img, cnt_img)
# If there are non-zero pixels left after "and"ing, there is an intersection
return np.any(anded)
def __start_of_tracking(self):
self.of_enabled = True
self.old_gray = cv2.cvtColor(realsensecam().bgr, cv2.COLOR_BGR2GRAY)
mask = np.zeros((realsensecam().H, realsensecam().W), np.uint8)
cv2.circle(mask, handdetector().fingertip_pos, 80, 255, -1)
cnt_mask = np.zeros_like(mask)
cv2.drawContours(cnt_mask, [handdetector().hand_cnt], 0, 255, -1)
mask = cv2.bitwise_and(mask, cnt_mask)
self.of_old_pts = p0 = cv2.goodFeaturesToTrack(
self.old_gray, mask=mask, **self.of_feature_params)
if self.of_old_pts is None:
self.__stop_of_tracking()
return
self.of_orig_existing_pts = self.of_old_pts.copy()
self.of_is_fingertip = np.array([0] * len(self.of_orig_existing_pts))
fingertip = np.array(handdetector().fingertip_pos)
for i, pt in enumerate(self.of_old_pts[:, 0]):
if np.linalg.norm(np.array(pt) - fingertip) < 30:
self.of_is_fingertip[i] = 1
def acquire_masks():
# Find the hand
_, depth_th_hand = cv2.threshold(realsensecam().depth_blurred, 1, 255,
cv2.THRESH_BINARY)
# Create the hand mask, the largest depth contour is assumed to be the hand
hand_cnt = None
hand_mask = np.zeros((realsensecam().H, realsensecam().W), np.uint8)
_, contours, _ = cv2.findContours(depth_th_hand, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
if len(contours) > 0:
largest_cnt = max(contours, key=lambda c: cv2.contourArea(c))
if cv2.contourArea(largest_cnt) > 500:
hand_cnt = largest_cnt
if hand_cnt is not None:
cv2.drawContours(hand_mask, [hand_cnt], 0, 255, cv2.FILLED)
# Create the shape mask
# Get mask by filtering by color saturation in HSV color space
hsv = cv2.cvtColor(realsensecam().bgr, cv2.COLOR_BGR2HSV)
shape_mask_unfiltered = cv2.inRange(hsv, np.array([0, 115, 0]),
np.array([255, 255, 255]))
_, shape_contours, _ = cv2.findContours(
cv2.bitwise_and(shape_mask_unfiltered,
shape_mask_unfiltered,
mask=(255 - hand_mask)), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
if len(shape_contours) == 0:
return None, None
largest_shape_cnt = max(shape_contours, key=lambda c: cv2.contourArea(c))
# Create a mask that should only contain the largest shape without hand
shape_mask_filtered = np.zeros_like(hand_mask)
cv2.drawContours(shape_mask_filtered, [largest_shape_cnt], 0, 255,
cv2.FILLED)
shape_mask_filtered[shape_mask_filtered != 0] = 1
return shape_mask_filtered, largest_shape_cnt
def next_frame(self):
self.frame += 1
self.publish('frame_begins', {'frame': self.frame})
if not realsensecam().acquire_frames():
return None
handdetector().determine_hand_cnt()
if handtracker().update() and not ui().menu_active:
detected_shapes = shapedetector().detect_shapes()
shapetracker().process_detected_shapes(detected_shapes)
touchedshapetracker().update()
if self.logger is not None:
self.logger.logAll()
return visualizer().visualize()
def __init__(self, cnt):
super().__init__()
# Calculate the bbox
bbox = Bbox(*cv2.boundingRect(cnt))
# Draw an isolated footprint of the shape
offset = tuple(- np.array(bbox.position()))
isolated = np.zeros(bbox.size(True), np.uint8)
cv2.drawContours(isolated, [cnt], 0, 255, -1, offset=offset)
footprint = cv2.copyMakeBorder(isolated, 15, 15, 15, 15, cv2.BORDER_CONSTANT, 0)
# Determine the color of the shape
x, y, w, h = bbox.xywh()
patch = realsensecam().bgr[y:y + h, x:x + w, :][int(h / 3):int(2 * h / 3), int(w / 3):int(2 * w / 3), :]
patch = cv2.GaussianBlur(patch, (51, 51), 0)
if patch is None:
color = (0, 0, 0)
else:
ph, pw, _ = patch.shape
color = patch[int(ph / 2), int(pw / 2)]
color = tuple([int(x) for x in color])
color_hsv = cv2.cvtColor(np.array([[color]], np.uint8), cv2.COLOR_BGR2HSV)[0][0]
self.cnt = cnt
self.bbox = bbox
self.color = color
self.color_hsv = color_hsv
self.footprint = footprint
self.angle = 0
self.state = 'fresh'
self.state_stable_since = shapetracker().epoch
self.pressed = False
self.moving = False
self.initial_swipe_xy = None
self.current_swipe_xy = None
self.initial_move_xy = None
self.current_move_xy = None
self.initial_degs = 0
self.current_degs = None
self.cnt_on_down = None
self.needs_transform_to_fit_shape = False
self.keypoints = {}
self.keypoints_on_down = None
self.action_name = ""
def __update_of(self):
if not self.of_enabled:
return
# Calculate optical flow
new_gray = cv2.cvtColor(realsensecam().bgr, cv2.COLOR_BGR2GRAY)
new_pts, st, err = cv2.calcOpticalFlowPyrLK(self.old_gray, new_gray,
self.of_old_pts, None,
**self.of_lk_params)
if new_pts is None:
self.__stop_of_tracking()
return
# Mark points that are not on the hand as invalid
for i, ptt in enumerate(new_pts):
if not handdetector().cnt_intersects_with_hand(
np.array([ptt]).astype(int)):
st[i][0] = 0
# Delete lost points
self.of_orig_existing_pts = self.of_orig_existing_pts[st == 1]
self.of_old_pts = self.of_old_pts[st == 1]
self.of_is_fingertip = self.of_is_fingertip[st.flatten() == 1]
new_pts = new_pts[st == 1]
if len(new_pts) == 0:
self.__stop_of_tracking()
return
old_finger_delta = self.finger_delta
old_finger_deg_delta = self.finger_deg_delta
T, R, t = icp.best_fit_transform(self.of_orig_existing_pts, new_pts)
self.finger_delta = tuple(t)
self.finger_deg_delta = np.rad2deg(np.arctan2(R[1, 0], R[0, 0]))
self.finger_transform = T
dt = np.linalg.norm(np.array(t) - np.array(old_finger_delta))
dr = abs(old_finger_deg_delta - self.finger_deg_delta)
# Prepare next iteration
self.old_gray = new_gray.copy()
self.of_old_pts = new_pts.reshape(-1, 1, 2)
self.of_orig_existing_pts = self.of_orig_existing_pts.reshape(-1, 1, 2)
return dt > 1 or dr > .3
def on_finger_pressing(self, _, data):
xy = data['fingertip_pos']
if self.menu_active:
self.menu.on_finger_pressing(xy)
elif self.action_menu_active:
self.action_menu.on_finger_pressing(xy)
elif 50 <= xy[0] <= realsensecam().W - 50 or xy[1] >= 50:
self.on_finger_up(None, data)
elif self.menu_armed:
self.menu_progress = min(1, time.time() - self.finger_down_ts)
if self.menu_progress >= 1:
self.__menu_open()
elif self.action_menu_armed:
self.action_menu_progress = min(1,
time.time() - self.finger_down_ts)
if self.action_menu_progress >= 1:
self.__action_menu_open()
else:
self.on_finger_down(None, data)
def visualize(self):
self.frame = realsensecam().bgr.copy()
if handdetector().hand_valid:
self.__shapes()
self.__touchets()
ui().visualize_menu(self) # Due to include cycle, this must be done in UI
self.__hand()
self.__shaperegionpicker()
else:
red_pic = np.full(self.frame.shape, (0, 0, 255), dtype=self.frame.dtype)
self.frame = cv2.addWeighted(self.frame, 1, red_pic, 0.5, 0)
# From here on, work with border
self.frame = cv2.copyMakeBorder(self.frame, 0, self.text_h + self.lost_shapes_h + 4, 0, 0, cv2.BORDER_CONSTANT, 0)
self.__lost_shapes()
self.__stats()
self.ts_of_last_frame = time.time()
return self.frame
def detect_shapes(self):
# Get mask by filtering by color saturation in HSV color space
hsv = cv2.cvtColor(realsensecam().bgr, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(
hsv, np.array([0, conf()['shape_saturation_threshold'], 0]),
np.array([255, 255, 255]))
self.most_recent_mask = mask
# Find contours in mask in order to isolate individual shapes
contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
detected_shapes = []
for cnt in contours:
if cv2.contourArea(cnt) < 750:
continue
if handdetector().cnt_intersects_with_hand(cnt):
continue
s = Shape(cnt)
if s is not None:
detected_shapes.append(s)
return detected_shapes
def determine_hand_cnt(self):
# Cut along the table surface to get all objects lying above it
_, depth_th_hand = cv2.threshold(realsensecam().depth_blurred,
conf()['hand_depth_threshold'], 255,
cv2.THRESH_BINARY)
self.most_recent_mask = depth_th_hand
# Detect the contours, the largest is assumed to be the hand
self.hand_cnt = None
self.secondary_hand_cnts = []
self.hand_valid = True
contours, _ = cv2.findContours(depth_th_hand, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
if len(contours) > 0:
sorted_cnts = sorted(contours, key=lambda c: -cv2.contourArea(c))
if cv2.contourArea(
sorted_cnts[0]) > 500: # Largest contour is the hand
self.hand_cnt = sorted_cnts[0]
for cnt in sorted_cnts[
1:]: # Other large contours will be saved as secondary hands
if cv2.contourArea(cnt) < 500:
break
self.secondary_hand_cnts.append(cnt)
if self.hand_cnt is None:
return # No hand detected
# The following code is used to find out where the hand enters the camera
# Initialize / reset datastructures
touched_corners = set(
) # This will hold the edges of the frame that are touched by the hand
self.edgepts = []
self.edgeextrem1 = None
self.edgeextrem2 = None
self.edgeextremcenter = None
# Detect points touching an edge and account which edges are touched
for pt in self.hand_cnt[:, 0]:
iscorner = False
if pt[1] <= 1: # Top edge
touched_corners.add(0)
iscorner = True
if pt[0] <= 1: # Left edge
touched_corners.add(1)
iscorner = True
if pt[1] >= realsensecam().H - 1: # Bottom edge
touched_corners.add(2)
iscorner = True
if pt[0] >= realsensecam().W - 1: # Right edge
touched_corners.add(3)
iscorner = True
if iscorner: # If the point has touched an edge, add it to edgepts
self.edgepts.append(pt)
# Make sure that top and bottom (or left and right) edge are not touched simultaneously
for i in touched_corners:
for j in touched_corners:
if i != j and i % 2 == j % 2:
# In this case, the hand is larger than the recorded area and we cannot infer anything
self.hand_valid = False
print("Hand error: Too long hand")
return
# Detect where the hand is touching the edge(s)
if len(self.edgepts) < 2:
self.hand_valid = False
print("Hand error: Edge points detection failed")
return
# Find the two points touching an edge that are furthest apart, as well as their center
pairwise_1_norm_dists = cdist(self.edgepts, self.edgepts, 'cityblock')
furthest_pts = np.unravel_index(np.argmax(pairwise_1_norm_dists),
pairwise_1_norm_dists.shape)
self.edgeextrem1 = tuple(self.edgepts[furthest_pts[0]])
self.edgeextrem2 = tuple(self.edgepts[furthest_pts[1]])
self.edgeextremcenter = (int(
(self.edgeextrem1[0] + self.edgeextrem2[0]) /
2), int((self.edgeextrem1[1] + self.edgeextrem2[1]) / 2))
# The following code is for finger detection
# Acquire slice
slice_img = self.__get_slice_img(3, 20)
if slice_img is None:
self.hand_valid = False
print("Hand error: Invalid slice img")
return
slice_cnts, _ = cv2.findContours(slice_img, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
if len(slice_cnts) == 0:
self.hand_valid = False
print("Hand error: Invalid slice contours")
return
# Get furthest point from arm entry point
cnts_arr = np.concatenate(slice_cnts)[:, 0]
dists = cdist(cnts_arr, [self.edgeextremcenter])
furthest_pt = cnts_arr[np.argmax(dists, axis=0)][0]
self.fingertip_pos = tuple(furthest_pt)
# Calculate height of fingertip
x = self.fingertip_pos[0]
y = self.fingertip_pos[1]
r = conf()['finger_height_measure_radius']
left = max(0, x - r)
top = max(0, y - r)
right = min(realsensecam().W, x + r)
bottom = min(realsensecam().H, y + r)
# Let the highest pixel of that surface be the fingertip height
cropped = realsensecam().depth_blurred[top:bottom, left:right]
observed_height = np.max(cropped)
if self.fingertip_height == np.inf:
self.fingertip_height = observed_height
else:
self.fingertip_height = 0.5 * observed_height + 0.5 * self.fingertip_height
import cv2
from conf import *
from controller import controller
from realsensecam import realsensecam
NAME = 'sample_videos/test'
if __name__ == "__main__":
with Conf():
# Initialize camera
realsensecam(list((NAME + '_rgb.mp4', NAME + '_depth.mp4')))
# Initialize controller
controller()
while True:
img = controller().next_frame()
if img is None:
print("Controller reports end of capture. Terminating.")
break
cv2.imshow('DynamicUIs', img)
k = cv2.waitKey(1)
if ord('q') == k:
break
else:
controller().on_key(k)
realsensecam().stop() # Stop camera
largest_shape_cnt = max(shape_contours, key=lambda c: cv2.contourArea(c))
# Create a mask that should only contain the largest shape without hand
shape_mask_filtered = np.zeros_like(hand_mask)
cv2.drawContours(shape_mask_filtered, [largest_shape_cnt], 0, 255,
cv2.FILLED)
shape_mask_filtered[shape_mask_filtered != 0] = 1
return shape_mask_filtered, largest_shape_cnt
def visualize_pts(img, pts, color):
for pt in pts.astype(int):
cv2.circle(img, tuple(pt), 1, color)
realsensecam()
rof = None
new_img = None
new_shape_mask_filtered = None
while True:
# Read a new frame from the camera
realsensecam().acquire_frames()
new_shape_mask_filtered, largest_shape_cnt = acquire_masks()
if new_shape_mask_filtered is None:
continue
new_img = cv2.cvtColor(realsensecam().bgr, cv2.COLOR_BGR2GRAY)
if rof is None:
rof = ReconstructiveOpticalFlow(new_img, new_shape_mask_filtered)
continue
'--breakpoints',
nargs='+',
type=int,
help="Pause at the specified frames and wait for a key to be pressed")
parser.add_argument('-l', '--logfile')
args = parser.parse_args()
if args.video_output is not None and args.video_input is not None:
print(
"Error: You may not use -i and -o simultaneously. Please use only up to one of them at a time."
)
exit(-1)
# Initialize camera
if args.video_input is not None:
realsensecam(list(filename_from_name(args.video_input)))
else:
realsensecam()
if args.logfile is not None:
logger = Logger()
else:
logger = None
controller(logger)
if args.video_output is not None:
videowriter = VideoWriter(*filename_from_name(args.video_output),
30, (realsensecam().W, realsensecam().H),
threaded=False)
breakpoints = args.b__breakpoints or []
def position_difference(self, other_shape):
return np.linalg.norm(other_shape.bbox.center_nparr() - self.bbox.center_nparr()) / realsensecam().diagonal