def callback(self, data):
try:
# capture image from the subscriber and convert to an OpenCV image
scene = self.bridge.imgmsg_to_cv2(data, "bgr8")
except CvBridgeError as e:
print(e)
if not self.voice_activated:
cv2.imshow("Object Recognition", imutils.resize(scene, height=700))
cv2.waitKey(1)
else:
self.current_frame += 1
if self.current_frame % self.frame_skip:
# Our operations on the frame come here
scene_kp, scene_des = self.detector.detectAndCompute(scene, None)
detected_objects = []
detected_objects_x = []
for obj_features in self.object_features:
obj_x = min(
[
point[
0
]
for point in self.detect_object(
obj_features, [
scene, scene_kp, scene_des
]
)
]
)
if obj_x != float('-inf'):
detected_objects.append(obj_features[0])
detected_objects_x.append(obj_x)
detected_objects = [
obj for (obj_x, obj) in sorted(zip(detected_objects_x, detected_objects))
]
for ob in self.objects:
if ob in self.previous_detected_objects and ob not in detected_objects:
self.object_counters[ob] = 0
if set(self.previous_detected_objects) == set(
detected_objects
) and not len(detected_objects) == 0:
self.consensus_counter += 1
else:
self.consensus_counter = 0
self.previous_detected_objects = detected_objects
cv2.imshow("Object Recognition", imutils.resize(scene, height=700))
cv2.waitKey(1)
if self.current_frame >= self.max_frames:
speak = 'I found '
if len(detected_objects) > 0:
for detected_object in detected_objects:
speak += detected_object + ', '
speak = speak[:-2]
speak = juditeutils.replace_right(speak, ', ', ' and ')
else:
speak += 'nothing'
print(speak)
self.consensus_counter = 0
cv2.imwrite(self.img_dir + '/objects' + str(datetime.now()) + '.jpg', scene)
self.pub.publish(','.join(detected_objects))
self.voicePub.publish(speak)
self.voice_activated = False
rospy.sleep(3)
self.current_frame = 0
def callback_tracking(self, data):
try:
# capture image from the subscriber and convert to an OpenCV image
scene = self.bridge.imgmsg_to_cv2(data, "bgr8")
except CvBridgeError as e:
print(e)
if self.first_frame is None:
self.first_frame = cv2.GaussianBlur(
cv2.cvtColor(scene, cv2.COLOR_BGR2GRAY), (21, 21), 0
)
return
if not self.voice_activated:
if rospy.get_param('~show_changes', False):
contours = self.track_changes(scene)
for cnt in contours:
cv2.rectangle(
scene, (cnt[0], cnt[1]), (cnt[0] + cnt[2], cnt[1] + cnt[3]), (0, 0, 255), 3
)
cv2.imshow("Object Recognition", imutils.resize(scene, height=700))
cv2.waitKey(1)
else:
self.current_frame += 1
if self.current_frame % self.frame_skip:
contours = self.track_changes(scene)
detected_objects = []
detected_objects_x = []
for cnt in contours:
gray_scene = cv2.cvtColor(scene, cv2.COLOR_BGR2GRAY)
possible_object = gray_scene[cnt[1]:cnt[1] + cnt[3], cnt[0]:cnt[0] + cnt[2]]
possible_object_kp, possible_object_des = self.detector.detectAndCompute(
possible_object, None
)
for obj_features in self.object_features:
homography, obj_rect = self.detect_object(
obj_features, [gray_scene, possible_object_kp, possible_object_des]
)
if homography is not None:
object_name = obj_features[0]
detected_objects.append(object_name)
detected_objects_x.append(cnt[0])
# displace the homography so it can be drawn
# on top of the object in the original image
for point in homography:
point[0] += cnt[0]
point[1] += cnt[1]
text_point_x = min([point[0] for point in homography]) if min(
[
point[0] for point in homography
]
) > 0 else max([point[0] for point in homography])
text_point = (text_point_x, min([point[1] for point in homography]))
# transform the homography so it can be used by the polylines function
homography = homography.reshape((-1, 1, 2))
cv2.polylines(scene, [homography], True, (0, 255, 255), 10)
cv2.imwrite(
self.img_dir + '/' + object_name + '_' + str(datetime.now()) +
'.jpg', scene
)
cv2.putText(
scene, object_name + ': ' + str(self.object_counters[object_name]),
text_point, cv2.FONT_HERSHEY_COMPLEX_SMALL, 1.2, (0, 0, 0), 2
)
detected_objects = [
obj for (obj_x, obj) in sorted(zip(detected_objects_x, detected_objects))
]
self.previous_detected_objects = detected_objects
cv2.imshow("Object Recognition", imutils.resize(scene, height=700))
cv2.waitKey(1)
if self.current_frame >= self.max_frames:
speak = 'I found '
if len(detected_objects) > 0:
for detected_object in detected_objects:
speak += detected_object + ', '
speak = speak[:-2]
speak = juditeutils.replace_right(speak, ', ', ' and ')
else:
speak += 'nothing'
print(speak)
self.consensus_counter = 0
cv2.imwrite(self.img_dir + '/objects' + str(datetime.now()) + '.jpg', scene)
self.pub.publish(detected_objects)
self.voicePub.publish(speak)
self.voice_activated = False
rospy.sleep(3)
self.current_frame = 0
def recognition_callback(self, data):
# gets the image from the camera and grays it out
try:
image = self.bridge.imgmsg_to_cv2(data, "bgr8")
except CvBridgeError as e:
print(e)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# detects faces
faces = self.face_detector.detectMultiScale(
gray,
scaleFactor=1.1,
minNeighbors=10,
minSize=(100, 100),
flags=cv2.cv.CV_HAAR_SCALE_IMAGE
)
people = []
unknowns = 0
for x, y, w, h in faces:
x1 = x + int(w * .1)
x2 = x1 + int(w * .8)
y1 = y + int(h * .2)
y2 = y1 + int(h * .8)
# isolates the detected face and uses our recognizer to classify it
roi_gray = gray[y1:y2, x1:x2]
cv2.rectangle(image, (x1, y1), (x2, y2), (0, 255, 0), 2)
class_text_point = (max(x1, 0), max(y1, 0))
confidence_text_point = (max(x1, 0), min(y2, gray.shape[1]))
# if the recognizer is either EigenFaces or FisherFaces, detected faces are resized
# before being classified, in the same way training images were
if self.recognizer_type in ['fisher', 'eigen']:
roi_gray = cv2.resize(roi_gray, (self.smallest_dimension, self.smallest_dimension))
[predicted_label, predicted_confidence] = self.recognizer.predict(roi_gray)
if predicted_confidence >= self.max_distance:
name = 'unknown'
unknowns += 1
else:
name = self.names[predicted_label]
people.append(name)
cv2.putText(image, 'Class: ' + name,
class_text_point, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.8, (0, 0, 0), 2)
cv2.putText(image, 'Confidence: ' + str(predicted_confidence),
confidence_text_point, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.8, (0, 0, 0), 2)
self.pub.publish(', '.join(people) if len(people) > 0 else None)
if self.voice_activated:
if len(people) + unknowns == 0:
self.voicePub.publish('I don\'t see anyone')
self.voice_activated = False
return
speak = 'I see '
if len(people) > 0:
speak += ', '.join(people)
if unknowns == 0:
speak = juditeutils.replace_right(speak, ', ', ' and ', 1)
else:
if len(people) > 0:
speak += ' and '
if unknowns == 1:
speak += 'an unknown person'
else:
speak += str(unknowns) + ' unknown people'
self.voicePub.publish(speak)
self.voice_activated = False
cv2.imshow('Face Recognition', imutils.resize(image, height=700))
cv2.waitKey(1)
