def test_roi_from_frame_center(self): r = Roi.from_frame(frame, SIDE.CENTER, 100) self.assertEqual(r, [0, 0, 640, 480]) r = Roi.from_frame(frame, SIDE.CENTER, 90) self.assertEqual(r, [(64 / 2), (48 / 2), 640 - 64, 480 - 48]) r = Roi.from_frame(frame, SIDE.CENTER, 10) self.assertEqual(r, [640 / 2 - 64 / 2, 480 / 2 - 48 / 2, 64, 48])
def test_roi_from_frame_left(self): r = Roi.from_frame(frame, SIDE.LEFT, 100) self.assertEqual(r, [0, 0, 640, 480]) r = Roi.from_frame(frame, SIDE.LEFT, 90) self.assertEqual(r, [0, 0, 640 - 64, 480]) r = Roi.from_frame(frame, SIDE.LEFT, 10) self.assertEqual(r, [0, 0, 64, 480])
def test_roi_from_frame_bottom(self): r = Roi.from_frame(frame, SIDE.BOTTOM, 100) self.assertEqual(r, [0, 0, 640, 480]) r = Roi.from_frame(frame, SIDE.BOTTOM, 90) self.assertEqual(r, [0, 48, 640, 480 - 48]) r = Roi.from_frame(frame, SIDE.BOTTOM, 10) self.assertEqual(r, [0, 480 - 48, 640, 48])
def test_roi_from_frame_right(self): r = Roi.from_frame(frame, SIDE.RIGHT, 100) self.assertEqual(r, [0, 0, 640, 480]) r = Roi.from_frame(frame, SIDE.RIGHT, 90) self.assertEqual(r, [64, 0, 640 - 64, 480]) r = Roi.from_frame(frame, SIDE.RIGHT, 10) self.assertEqual(r, [640 - 64, 0, 64, 480])
def test_roi_from_frame_top(self): """ Test that check the creation of roi from frames """ r = Roi.from_frame(frame, SIDE.TOP, 100) self.assertEqual(r, [0, 0, 640, 480]) r = Roi.from_frame(frame, SIDE.TOP, 90) self.assertEqual(r, [0, 0, 640, 480 - 48]) r = Roi.from_frame(frame, SIDE.TOP, 10) self.assertEqual(r, [0, 0, 640, 48])
def get_roi_to_use(self, frame): """ Calculate the roi to be used depending on the situation of the hand (initial, detected, tracked) :param frame: :return: """ current_roi = None if self._detected: current_roi = self.detection_roi else: # if we already have failed to detect we use the extended_roi if self._consecutive_detection_fails > 0: if self._tracked: current_roi = self.tracking_roi else: current_roi = self.extended_roi else: # Not detected and not consecutive fails on detection. # It's probably the first time we try to detect. # If no initial_roi is given an square of 200 x 200 is taken on the center if self.initial_roi is not None and self.initial_roi != Roi(): current_roi = self.initial_roi else: current_roi = Roi.from_frame(frame, SIDE.CENTER, 50) assert current_roi != Roi(), "hand can't be detected on a %s roi of the frame" % str(current_roi) return current_roi
def test_hand_detect(self): """ Test that check the creation of roi from frames """ hand = Hand() hand.depth_threshold = 130 expected_results = { "20190725130248.png": [False, False], "20190725130249.png": [False, False], "20190725130250.png": [True, True], "20190725130251.png": [False, True], # fingers too close and it's considered not a hand "20190725130252.png": [True, True], "20190725130253.png": [True, True], "20190725130254.png": [False, True], "20190725130255.png": [False, True], "20190725130256.png": [False, True], "20190725130257.png": [False, True], "20190725130258.png": [False, True] } full_path = "/home/robolab/robocomp/components/robocomp-robolab/components/detection/handDetection/src/images/depth_images" for file in sorted(os.listdir(full_path)): if file.endswith(".png") and file in expected_results: frame = RGBDFrame(cv2.imread(os.path.join(full_path, file), 0)) hand.initial_roi = Roi.from_frame(frame, SIDE.CENTER, 50) hand.detect_and_track(frame) print("testing file %s" % file) self.assertEqual(hand.detected, expected_results[file][0]) self.assertEqual(hand.tracked, expected_results[file][1]) frame = self.draw_in_frame(hand, frame) cv2.imshow("final", frame) key = cv2.waitKey(5000) if key == 112: while cv2.waitKey(1000) != 112: pass
def update_hand_with_contour(self, hand_contour): """ Attributes of the hand are calculated from the hand contour. TODO: calculate a truth value A score of 100 is the maximum value for the hand truth. This value is calculated like this: A hand is expected to have 5 finger tips, 4 intertips, a center of mass :param hand_contour: calculated contour that is expected to describe a hand :return: None """ hull2 = cv2.convexHull(hand_contour, returnPoints=False) # Get defect points defects = cv2.convexityDefects(hand_contour, hull2) if defects is not None: estimated_fingertips_coords, \ estimated_fingertips_indexes, \ estimated_intertips_coords, \ estimated_intertips_indexes = self._calculate_fingertips(hand_contour, defects) is_hand = self.is_hand(estimated_fingertips_coords, estimated_intertips_coords, strict=True) if is_hand: self._fingertips = estimated_fingertips_coords self._intertips = estimated_intertips_coords if len(estimated_fingertips_coords) == 5: fingers_contour = np.take(hand_contour, estimated_fingertips_indexes + estimated_intertips_indexes, axis=0, mode="wrap") bounding_rect, hand_circle, self._contour = self.get_hand_bounding_rect_from_fingers( hand_contour, fingers_contour) # detection roi is set to the bounding rect of the fingers upscaled 20 pixels # self.detection_roi = Roi(bounding_rect) self.detection_roi = Roi(bounding_rect).upscaled(Roi.from_frame(self._last_frame, SIDE.CENTER, 100), 10) if self._debug: to_show = self._last_frame.copy() cv2.drawContours(to_show, [hand_contour], -1, (255, 255, 255), 2) cv2.drawContours(to_show, [fingers_contour], -1, (200, 200, 200), 2) to_show = self.detection_roi.draw_on_frame(to_show) # cv2.rectangle(to_show, (self.detection_roi.y, self.detection_roi.x), (self.detection_roi.y + self.detection_roi.height, self.detection_roi.x + self.detection_roi.width), [255, 255, 0]) # (x, y, w, h) = cv2.boundingRect(hand_contour) # cv2.rectangle(to_show, (self.detection_roi.y, self.detection_roi.x), (self.detection_roi.x + self.detection_roi.height, self.detection_roi.x + self.detection_roi.width), [255, 255, 0]) cv2.imshow("update_hand_with_contour", to_show) self._detected = True self._detection_status = 1 self._ever_detected = True self._confidence = 100 else: self._detection_status = -1 self._detected = False self._confidence = 0 return else: self._detection_status = -1 self._detected = False self._confidence = 0 return # Find moments of the largest contour moments = cv2.moments(hand_contour) center_of_mass = None finger_distances = [] average_defect_distance = None # Central mass of first order moments if moments['m00'] != 0: cx = int(moments['m10'] / moments['m00']) # cx = M10/M00 cy = int(moments['m01'] / moments['m00']) # cy = M01/M00 center_of_mass = (cx, cy) self._center_of_mass = center_of_mass self._position_history.append(center_of_mass) if center_of_mass is not None and len(estimated_intertips_coords) > 0: # Distance from each finger defect(finger webbing) to the center mass distance_between_defects_to_center = [] for far in estimated_intertips_coords: x = np.array(far) center_mass_array = np.array(center_of_mass) distance = np.sqrt( np.power(x[0] - center_mass_array[0], 2) + np.power(x[1] - center_mass_array[1], 2) ) distance_between_defects_to_center.append(distance) # Get an average of three shortest distances from finger webbing to center mass sorted_defects_distances = sorted(distance_between_defects_to_center) average_defect_distance = np.mean(sorted_defects_distances[0:2]) self._average_defect_distance = average_defect_distance # # Get fingertip points from contour hull # # If points are in proximity of 80 pixels, consider as a single point in the group # finger = [] # for i in range(0, len(hull) - 1): # if (np.absolute(hull[i][0][0] - hull[i + 1][0][0]) > 10) or ( # np.absolute(hull[i][0][1] - hull[i + 1][0][1]) > 10): # if hull[i][0][1] < 500: # finger.append(hull[i][0]) # # # # The fingertip points are 5 hull points with largest y coordinates # finger = sorted(finger, key=lambda x: x[1]) # fingers = finger[0:5] if center_of_mass is not None and len(estimated_fingertips_coords) > 0: # Calculate distance of each finger tip to the center mass finger_distances = [] for i in range(0, len(estimated_fingertips_coords)): distance = np.sqrt( np.power(estimated_fingertips_coords[i][0] - center_of_mass[0], 2) + np.power( estimated_fingertips_coords[i][1] - center_of_mass[0], 2)) finger_distances.append(distance) self._finger_distances = finger_distances else: self._detection_status = -2 self._detected = False self._confidence = 0 return