def process_frame_for_line(frame, image_format="PIL", process_image_width=320):
cv2 = import_opencv()
cv_frame = ImageFunctions.convert(frame, format="OpenCV")
from imutils import resize
resized_frame = resize(cv_frame, width=process_image_width)
hsv_lower, hsv_upper = calculate_blue_limits()
image_mask = color_mask(resized_frame, hsv_lower, hsv_upper)
line_contour = find_largest_contour(image_mask)
centroid = None
scaled_image_centroid = None
rectangle_dimensions = None
angle = None
if line_contour is not None:
# find centroid of contour
scaled_image_centroid = find_centroid(line_contour)
centroid = center_reposition(scaled_image_centroid, resized_frame)
bounding_rectangle = cv2.boundingRect(line_contour)
rectangle_dimensions = bounding_rectangle[2:5]
angle = get_object_target_lock_control_angle(centroid, resized_frame)
robot_view_img = robot_view(resized_frame, image_mask, line_contour,
scaled_image_centroid)
if image_format.lower() != "opencv":
robot_view_img = ImageFunctions.convert(robot_view_img, format="PIL")
return DotDict({
"line_center": centroid,
"robot_view": robot_view_img,
"rectangle_dimensions": rectangle_dimensions,
"angle": angle,
})
def color_filter(self, frame, color: str = "red"):
frame = ImageFunctions.convert(frame, format="OpenCV")
mask = self.__get_color_mask(frame, color)
filtered_image = cv2.bitwise_and(frame, frame, mask=mask)
if self.format.lower() == "pil":
filtered_image = ImageFunctions.convert(mask, format="PIL")
return filtered_image
def __get_processed_current_frame(self):
image = self.__frame_handler.frame
if self.format.lower() == "opencv":
image = ImageFunctions.convert(image, format="opencv")
return image
def __call__(self, frame, color: Union[str, list] = "red"):
def parse_colors(color_arg):
colors = []
if type(color_arg) == str:
if color_arg not in VALID_COLORS:
raise ValueError(
f"Valid color values are {', '.join(VALID_COLORS[:-1])} or {VALID_COLORS[-1]}"
)
colors = [color_arg]
elif type(color_arg) in (list, tuple):
if not set(color_arg).issubset(VALID_COLORS):
raise ValueError(
f"Valid color values are {', '.join(VALID_COLORS[:-1])} or {VALID_COLORS[-1]}"
)
colors = color_arg
if len(colors) > 3:
raise ValueError("Cannot pass more than three colors.")
return colors
import_libs()
frame = ImageFunctions.convert(frame, format="OpenCV")
if self._frame_scaler is None:
_, width = frame.shape[0:2]
self._frame_scaler = width / self._image_processing_width
for c in parse_colors(color):
self.balls[c] = self.__find_most_likely_ball(
ball=self.balls.get(c), frame=frame, color=c)
robot_view = frame.copy()
ball_data = DotDict({})
for ball_color, ball_object in self.balls.items():
ball_data[ball_color] = ball_object
self.__draw_ball_contrail(robot_view, ball_object)
if ball_object.found:
self.__draw_ball_position(robot_view, ball_object)
ball_data["robot_view"] = ImageFunctions.convert(
robot_view, self.format)
if self._print_fps:
self._fps.update()
return ball_data
def __prepare_face_data(self, frame, face, rectangle, center, features):
face.original_detection_frame = frame
if center is None:
face.clear()
face.robot_view = ImageFunctions.convert(frame,
format=self._format)
return face
# resize back to original frame resolution
face.rectangle = tuple(
(int(item * self._frame_scaler) for item in rectangle))
face.center_default = tuple(
(int(item * self._frame_scaler) for item in center))
face.features = (features * self._frame_scaler).astype("int")
face.robot_view = ImageFunctions.convert(self.__draw_on_frame(
frame=frame.copy(), face=face),
format=self._format)
return face
def process(self, frame):
if isinstance(self.__format,
str) and self.__format.lower() == "opencv":
frame = ImageFunctions.convert(frame, format="opencv")
if self.__elapsed_frames % self.__frame_interval == 0:
if self.callback_has_argument:
self.__event_executor.submit(self.__generic_action_callback,
frame)
else:
self.__event_executor.submit(self.__generic_action_callback)
self.__elapsed_frames = (self.__elapsed_frames +
1) % self.__frame_interval
def __call__(self, face):
frame = ImageFunctions.convert(face.original_detection_frame.copy(),
format="OpenCV")
if not face.found:
self.emotion.clear()
self.emotion.robot_view = frame
return self.emotion
self.emotion = self.__get_emotion(frame=frame,
face=face,
emotion=self.emotion)
return self.emotion
def play_animated_image_file(self,
file_path_or_url,
background=False,
loop=False):
"""Render an animated image to the screen from a file or URL.
:param str file_path_or_url: A file path or URL to the image
:param bool background: Set whether the image should be in a background thread
or in the main thread.
:param bool loop: Set whether the image animation should start again when it
has finished
"""
image = ImageFunctions.get_pil_image_from_path(file_path_or_url)
self.play_animated_image(image, background, loop)
def display_image(self, image, xy=None, invert=False):
"""Render a static image to the screen from a file or URL at a given
position.
The image should be provided as a PIL Image object.
:param Image image: A PIL Image object to be rendered
:param tuple xy: The position on the screen to render the image. If not
provided or passed as `None` the image will be drawn in the top-left of
the screen.
:param bool invert: Set to True to flip the on/off state of each pixel in the image
"""
self.__display(
self.assistant.process_image(
ImageFunctions.convert(image, format="PIL")),
invert=invert,
)
def __call__(self, frame):
"""Detect a face in an image frame.
:param frame: Image frame in OpenCV or PIL format.
:return: Face object containing data about the detected face.
"""
self.__import_libs()
frame = ImageFunctions.convert(frame, format="OpenCV")
if self._frame_scaler is None:
_, width = frame.shape[0:2]
self._frame_scaler = (width / self._image_processing_width
if self._image_processing_width is not None
else 1)
frame_to_process = self.__get_frame_to_process(frame=frame)
if self._face_tracker is None:
# if the face tracker has not been started, we first need to use face detection to find a face
face_rectangle, face_center, face_features = self.__detect_largest_face(
frame=frame_to_process)
if face_center is not None and self._enable_tracking:
# Face found, enable dlib correlation tracker for subsequent calls
self.__start_tracker(frame=frame_to_process,
rectangle=face_rectangle)
else:
# We are in face tracking mode, use dlib correlation tracker to track the face
face_rectangle, face_center, face_features = self.__track_face(
frame=frame_to_process)
if face_center is None:
self.__stop_tracker()
# attempt to detect face since tracker has failed
face_rectangle, face_center, face_features = self.__detect_largest_face(
frame=frame_to_process)
self.face = self.__prepare_face_data(
frame=frame,
face=self.face,
rectangle=face_rectangle,
center=face_center,
features=face_features,
)
if self._print_fps:
self._fps.update()
return self.face
def display_image_file(self, file_path_or_url, xy=None, invert=False):
"""Render a static image to the screen from a file or URL at a given
position.
The display's positional properties (e.g. `top_left`, `top_right`) can be used to assist with
specifying the `xy` position parameter.
:param str file_path_or_url: A file path or URL to the image
:param tuple xy: The position on the screen to render the image. If not
provided or passed as `None` the image will be drawn in the top-left of
the screen.
:param bool invert: Set to True to flip the on/off state of each pixel in the image
"""
self.display_image(
ImageFunctions.get_pil_image_from_path(file_path_or_url),
xy=xy,
invert=invert,
)
def __get_emotion(self, frame, face, emotion):
"""Emotion detection is carried out by taking the 68 face feature
landmark positions found using the dlib landmark detector and putting
them through a trained SVC model (in onnx format) that predicts the
most likely emotion. The prediction outputs probabilities for each
emotion type which are then put through a moving average filter to
smooth the output. If being used for non-realtime applications (on
static images) then the apply_mean_filter class attribute should be set
to False.
:param frame: Original camera frame used to detect the face (in OpenCV format), used for drawing robot_view.
:param face: Face object obtained from FaceDetector
:param emotion: Emotion object to store the resulting prediction data
:return: Emotion object that was passed into this function.
"""
def get_svc_feature_vector(features, face_angle):
"""The 68 face feature landmark positions need to be put into the
same format as was used to train the SVC model. The basic process
is as follows:
1. Use the face angle to apply a rotation matrix to orient the face features so that the eyes lie on a
horizontal line.
2. Find the mean (x, y) coordinate of the resulting face features so we can transform the (x, y)
face feature coordinates to be zero-centered.
3. Find the interpupillary distance from the left and right eye center to normalize the (x, y)
coordinates so that the resulting face features are independent of face scale.
Note: this will not normalize the face features to lie between 0 and 1 as is typically done when
pre-processing data to go into a machine learning algorithm. Previously, the diagonal of the face
rectangle was used to accomplish this but because the face rectangle from face detection comes in
discreet sizes (based on the pyramid layers), this had poor performance because a continuous scaling
factor is desired. Interpupillary distance is a good metric since the variance across humans is quite
small, and the data used for training contains a wide selection of face types to accommodate for this.
Sklearn's StandardScaler() was added to the SVC pipeline to ensure the data lies between 0 and 1 before
the classification is carried out - this does not completely remove the need for scaling here as was
found during testing.
4. After rotation, transformation and scaling, the resulting (x, y) coordinates are flatted into a numpy
array in the form array([[x1, y1, x2, y2, ... x68, y68]]) with shape (1, 136). This is now in the format
to send to the onnx SVC model for emotion classification.
:param features: 68 landmark face feature (x, y) coordinates in a 68x2 numpy array (found from FaceDetector)
:param face_angle: face angle from Face object (found from FaceDetector).
:return: 1x136 feature vector to put through the onnx SVC model.
"""
rotation_matrix = np.array([
[
np.cos(np.radians(face_angle)),
-np.sin(np.radians(face_angle))
],
[
np.sin(np.radians(face_angle)),
np.cos(np.radians(face_angle))
],
])
face_features_rotated = rotation_matrix.dot(features.T).T
face_feature_mean = face_features_rotated.mean(axis=0)
left_eye_center = np.mean(
face_features_rotated[self.left_eye_start:self.left_eye_end],
axis=0)
right_eye_center = np.mean(
face_features_rotated[self.right_eye_start:self.right_eye_end],
axis=0)
interpupillary_distance = np.linalg.norm(left_eye_center -
right_eye_center)
feature_vector = []
for landmark in face_features_rotated:
relative_vector = (landmark -
face_feature_mean) / interpupillary_distance
feature_vector.append(relative_vector[0])
feature_vector.append(relative_vector[1])
return np.asarray([feature_vector])
if len(face.features) != 68:
raise ValueError(
"This function is only compatible with dlib's 68 landmark feature predictor."
)
X = get_svc_feature_vector(face.features, face.angle)
# Run feature vector through onnx model and convert results to a numpy array
probabilities = np.asarray(
list(
self._emotion_model.run(
None, {self._onnx_input_node_name: X.astype(np.float32)
})[1][0].values()))
if self._apply_mean_filter:
self._probability_mean_array, probabilities = running_mean(
self._probability_mean_array, probabilities)
# Get the index of the most likely emotion type and associate to corresponding emotion string
max_index = int(np.argmax(probabilities))
emotion.type = self.emotion_types[max_index]
emotion.confidence = round(probabilities[max_index], 2)
emotion.robot_view = ImageFunctions.convert(
self.__draw_on_frame(frame=frame.copy(),
face=face,
emotion=self.emotion),
format=self._format,
)
return emotion
def format(self, format_value):
ImageFunctions.image_format_check(format_value)
self._format = format_value.lower()
def __init__(self, path: str):
self.path = path
self.data = ImageFunctions.get_pil_image_from_path(self.path)
if self.data is None:
raise AttributeError(f"Couldn't load image {path}")