def callback(self, data): try: # capture image from the subscriber and convert to an OpenCV image scene = self.bridge.imgmsg_to_cv2(data, "bgr8") except CvBridgeError as e: print(e) if not self.voice_activated: cv2.imshow("Object Recognition", imutils.resize(scene, height=700)) cv2.waitKey(1) else: self.current_frame += 1 if self.current_frame % self.frame_skip: # Our operations on the frame come here scene_kp, scene_des = self.detector.detectAndCompute(scene, None) detected_objects = [] detected_objects_x = [] for obj_features in self.object_features: obj_x = min( [ point[ 0 ] for point in self.detect_object( obj_features, [ scene, scene_kp, scene_des ] ) ] ) if obj_x != float('-inf'): detected_objects.append(obj_features[0]) detected_objects_x.append(obj_x) detected_objects = [ obj for (obj_x, obj) in sorted(zip(detected_objects_x, detected_objects)) ] for ob in self.objects: if ob in self.previous_detected_objects and ob not in detected_objects: self.object_counters[ob] = 0 if set(self.previous_detected_objects) == set( detected_objects ) and not len(detected_objects) == 0: self.consensus_counter += 1 else: self.consensus_counter = 0 self.previous_detected_objects = detected_objects cv2.imshow("Object Recognition", imutils.resize(scene, height=700)) cv2.waitKey(1) if self.current_frame >= self.max_frames: speak = 'I found ' if len(detected_objects) > 0: for detected_object in detected_objects: speak += detected_object + ', ' speak = speak[:-2] speak = juditeutils.replace_right(speak, ', ', ' and ') else: speak += 'nothing' print(speak) self.consensus_counter = 0 cv2.imwrite(self.img_dir + '/objects' + str(datetime.now()) + '.jpg', scene) self.pub.publish(','.join(detected_objects)) self.voicePub.publish(speak) self.voice_activated = False rospy.sleep(3) self.current_frame = 0
def callback_tracking(self, data): try: # capture image from the subscriber and convert to an OpenCV image scene = self.bridge.imgmsg_to_cv2(data, "bgr8") except CvBridgeError as e: print(e) if self.first_frame is None: self.first_frame = cv2.GaussianBlur( cv2.cvtColor(scene, cv2.COLOR_BGR2GRAY), (21, 21), 0 ) return if not self.voice_activated: if rospy.get_param('~show_changes', False): contours = self.track_changes(scene) for cnt in contours: cv2.rectangle( scene, (cnt[0], cnt[1]), (cnt[0] + cnt[2], cnt[1] + cnt[3]), (0, 0, 255), 3 ) cv2.imshow("Object Recognition", imutils.resize(scene, height=700)) cv2.waitKey(1) else: self.current_frame += 1 if self.current_frame % self.frame_skip: contours = self.track_changes(scene) detected_objects = [] detected_objects_x = [] for cnt in contours: gray_scene = cv2.cvtColor(scene, cv2.COLOR_BGR2GRAY) possible_object = gray_scene[cnt[1]:cnt[1] + cnt[3], cnt[0]:cnt[0] + cnt[2]] possible_object_kp, possible_object_des = self.detector.detectAndCompute( possible_object, None ) for obj_features in self.object_features: homography, obj_rect = self.detect_object( obj_features, [gray_scene, possible_object_kp, possible_object_des] ) if homography is not None: object_name = obj_features[0] detected_objects.append(object_name) detected_objects_x.append(cnt[0]) # displace the homography so it can be drawn # on top of the object in the original image for point in homography: point[0] += cnt[0] point[1] += cnt[1] text_point_x = min([point[0] for point in homography]) if min( [ point[0] for point in homography ] ) > 0 else max([point[0] for point in homography]) text_point = (text_point_x, min([point[1] for point in homography])) # transform the homography so it can be used by the polylines function homography = homography.reshape((-1, 1, 2)) cv2.polylines(scene, [homography], True, (0, 255, 255), 10) cv2.imwrite( self.img_dir + '/' + object_name + '_' + str(datetime.now()) + '.jpg', scene ) cv2.putText( scene, object_name + ': ' + str(self.object_counters[object_name]), text_point, cv2.FONT_HERSHEY_COMPLEX_SMALL, 1.2, (0, 0, 0), 2 ) detected_objects = [ obj for (obj_x, obj) in sorted(zip(detected_objects_x, detected_objects)) ] self.previous_detected_objects = detected_objects cv2.imshow("Object Recognition", imutils.resize(scene, height=700)) cv2.waitKey(1) if self.current_frame >= self.max_frames: speak = 'I found ' if len(detected_objects) > 0: for detected_object in detected_objects: speak += detected_object + ', ' speak = speak[:-2] speak = juditeutils.replace_right(speak, ', ', ' and ') else: speak += 'nothing' print(speak) self.consensus_counter = 0 cv2.imwrite(self.img_dir + '/objects' + str(datetime.now()) + '.jpg', scene) self.pub.publish(detected_objects) self.voicePub.publish(speak) self.voice_activated = False rospy.sleep(3) self.current_frame = 0
def recognition_callback(self, data): # gets the image from the camera and grays it out try: image = self.bridge.imgmsg_to_cv2(data, "bgr8") except CvBridgeError as e: print(e) gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) # detects faces faces = self.face_detector.detectMultiScale( gray, scaleFactor=1.1, minNeighbors=10, minSize=(100, 100), flags=cv2.cv.CV_HAAR_SCALE_IMAGE ) people = [] unknowns = 0 for x, y, w, h in faces: x1 = x + int(w * .1) x2 = x1 + int(w * .8) y1 = y + int(h * .2) y2 = y1 + int(h * .8) # isolates the detected face and uses our recognizer to classify it roi_gray = gray[y1:y2, x1:x2] cv2.rectangle(image, (x1, y1), (x2, y2), (0, 255, 0), 2) class_text_point = (max(x1, 0), max(y1, 0)) confidence_text_point = (max(x1, 0), min(y2, gray.shape[1])) # if the recognizer is either EigenFaces or FisherFaces, detected faces are resized # before being classified, in the same way training images were if self.recognizer_type in ['fisher', 'eigen']: roi_gray = cv2.resize(roi_gray, (self.smallest_dimension, self.smallest_dimension)) [predicted_label, predicted_confidence] = self.recognizer.predict(roi_gray) if predicted_confidence >= self.max_distance: name = 'unknown' unknowns += 1 else: name = self.names[predicted_label] people.append(name) cv2.putText(image, 'Class: ' + name, class_text_point, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.8, (0, 0, 0), 2) cv2.putText(image, 'Confidence: ' + str(predicted_confidence), confidence_text_point, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.8, (0, 0, 0), 2) self.pub.publish(', '.join(people) if len(people) > 0 else None) if self.voice_activated: if len(people) + unknowns == 0: self.voicePub.publish('I don\'t see anyone') self.voice_activated = False return speak = 'I see ' if len(people) > 0: speak += ', '.join(people) if unknowns == 0: speak = juditeutils.replace_right(speak, ', ', ' and ', 1) else: if len(people) > 0: speak += ' and ' if unknowns == 1: speak += 'an unknown person' else: speak += str(unknowns) + ' unknown people' self.voicePub.publish(speak) self.voice_activated = False cv2.imshow('Face Recognition', imutils.resize(image, height=700)) cv2.waitKey(1)