def __init__(self, args, camera, resolution=(320, 240), framerate=32):
"""Initialization method of :class:`t_system.vision.Vision` class.
Args:
camera: Camera object from PiCamera.
resolution: rPi camera's resolution data.
framerate: rPi camera's framerate data.
"""
if not args["cascadefile"]:
ccade_xml_file = T_SYSTEM_PATH + "/haarcascade/frontalface_default.xml"
self.decider = Decider()
else:
ccade_xml_file = T_SYSTEM_PATH + "/haarcascade/" + args["cascadefile"] + ".xml"
self.decider = Decider(args["cascadefile"])
self.servo_pan = Motor(17, self.decider) # pan means rotate right and left ways.
self.servo_tilt = Motor(14, self.decider, 4.5, False) # tilt means rotate up and down ways.
self.aimer = Aimer()
self.show_stream = args["show_stream"] # 'show-stream' argument automatically converted this type.
self.augmented = args["augmented"]
self.camera = camera
self.camera.resolution = resolution
self.camera.framerate = framerate
# self.camera.start_preview()
self.rawCapture = PiRGBArray(camera, size=resolution)
self.object_cascade = cv2.CascadeClassifier(ccade_xml_file)
(self.frame_width, self.frame_height) = resolution
self.mqtt_receimitter = None
# allow the camera to warmup
time.sleep(0.1)
def __init__(self, args):
"""Initialization method of :class:`t_system.vision.Vision` class.
Args:
args: Command-line arguments.
"""
self.detection_model = args["detection_model"]
if self.detection_model == "haarcascade":
self.detect_things = self.detect_with_haarcascade
else:
self.detect_things = self.detect_with_hog_or_cnn
if args['use_tracking_api']:
self.detect_track = self.d_t_with_cv_ta
else:
self.detect_track = self.d_t_without_cv_ta
self.no_recognize = args["no_recognize"]
if not self.no_recognize:
encoding_pickle_file = T_SYSTEM_PATH + "/recognition_encodings/" + args[
"encoding_file"] + ".pickle"
self.recognition_data = pickle.loads(
open(encoding_pickle_file, "rb").read())
self.track = self.track_with_recognizing
else:
self.track = self.track_without_recognizing
# Specify the tracker type
self.tracker_type = args["tracker_type"]
self.hearer = Hearer(args)
# self.hearer = None
resolution = (args["resolution"][0], args["resolution"][0])
self.camera = PiCamera()
self.camera.resolution = resolution
self.camera.framerate = args["framerate"]
self.recorder = Recorder(self.camera, self.hearer)
self.rawCapture = PiRGBArray(self.camera, size=resolution)
ccade_xml_file = T_SYSTEM_PATH + "/haarcascade/" + args[
"cascade_file"] + ".xml"
self.object_cascade = cv2.CascadeClassifier(ccade_xml_file)
(self.frame_width, self.frame_height) = resolution
self.decider = None
if args["AI"] == "official_ai":
self.decider = Decider(args["cascade_file"])
self.target_locker = LockingSystem(args, resolution, self.decider)
self.arm = None
if args["robotic_arm"]:
self.arm = Arm(args["robotic_arm"])
self.aimer = Aimer()
self.show_stream = args[
"show_stream"] # 'show-stream' argument automatically converted this type.
self.mark_object = self.get_mark_object(args["found_object_mark"])
self.record = args["record"]
self.record_path = ""
self.record_name = ""
self.augmented = False
if args["interface"] == "augmented":
self.augmented = True
self.mqtt_receimitter = None
self.current_frame = None
# Allow the camera to warm up
time.sleep(0.1)
class Vision:
"""Class to define a vision of tracking system..
This class provides necessary initiations and functions named :func:`t_system.vision.Vision.detect_track`
as the loop for each camera frames for tracking mode, named :func:`t_system.vision.Vision.learn` as the
learning ability and :func:`t_system.vision.Vision.security` as the security mode.
"""
def __init__(self, args):
"""Initialization method of :class:`t_system.vision.Vision` class.
Args:
args: Command-line arguments.
"""
self.detection_model = args["detection_model"]
if self.detection_model == "haarcascade":
self.detect_things = self.detect_with_haarcascade
else:
self.detect_things = self.detect_with_hog_or_cnn
if args['use_tracking_api']:
self.detect_track = self.d_t_with_cv_ta
else:
self.detect_track = self.d_t_without_cv_ta
self.no_recognize = args["no_recognize"]
if not self.no_recognize:
encoding_pickle_file = T_SYSTEM_PATH + "/recognition_encodings/" + args[
"encoding_file"] + ".pickle"
self.recognition_data = pickle.loads(
open(encoding_pickle_file, "rb").read())
self.track = self.track_with_recognizing
else:
self.track = self.track_without_recognizing
# Specify the tracker type
self.tracker_type = args["tracker_type"]
self.hearer = Hearer(args)
# self.hearer = None
resolution = (args["resolution"][0], args["resolution"][0])
self.camera = PiCamera()
self.camera.resolution = resolution
self.camera.framerate = args["framerate"]
self.recorder = Recorder(self.camera, self.hearer)
self.rawCapture = PiRGBArray(self.camera, size=resolution)
ccade_xml_file = T_SYSTEM_PATH + "/haarcascade/" + args[
"cascade_file"] + ".xml"
self.object_cascade = cv2.CascadeClassifier(ccade_xml_file)
(self.frame_width, self.frame_height) = resolution
self.decider = None
if args["AI"] == "official_ai":
self.decider = Decider(args["cascade_file"])
self.target_locker = LockingSystem(args, resolution, self.decider)
self.arm = None
if args["robotic_arm"]:
self.arm = Arm(args["robotic_arm"])
self.aimer = Aimer()
self.show_stream = args[
"show_stream"] # 'show-stream' argument automatically converted this type.
self.mark_object = self.get_mark_object(args["found_object_mark"])
self.record = args["record"]
self.record_path = ""
self.record_name = ""
self.augmented = False
if args["interface"] == "augmented":
self.augmented = True
self.mqtt_receimitter = None
self.current_frame = None
# Allow the camera to warm up
time.sleep(0.1)
def d_t_without_cv_ta(self, stop_thread, format='bgr'):
"""The low-level method to provide detecting and tracking objects without using OpenCV's tracking API.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
if self.record:
self.recorder.start("track")
self.detect_initiate()
for frame in self.camera.capture_continuous(self.rawCapture,
format=format,
use_video_port=True):
# grab the raw NumPy array representing the image, then initialize the timestamp and occupied/unoccupied text
frame = frame.array
self.current_frame = frame.copy(
) # For reaching with overwrite privileges.
rgb, detected_boxes = self.detect_things(self.current_frame)
reworked_boxes = self.relocate_detected_coords(detected_boxes)
if not self.no_recognize:
names = self.recognize_things(rgb, detected_boxes)
else:
names = None
self.track(self.current_frame, reworked_boxes, names)
# time.sleep(0.1) # Allow the servos to complete the moving.
self.show_frame(self.current_frame)
self.truncate_stream()
# print("frame showed!")
if self.check_loop_ended(stop_thread):
break
def d_t_with_cv_ta(self, stop_thread, format='bgr'):
"""The low-level method to provide detecting and tracking objects with using OpenCV's tracking API.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
if self.record:
self.recorder.start("track")
tracked_boxes = [] # this became array. because of overriding.
names = []
multi_tracker = cv2.MultiTracker_create()
rgb, detected_boxes = self.detect_initiate()
found_count = 0
d_t_failure_count = 0
use_detection = 0
for frame in self.camera.capture_continuous(self.rawCapture,
format=format,
use_video_port=True):
if len(detected_boxes) > len(tracked_boxes):
if not self.no_recognize:
names = self.recognize_things(rgb, detected_boxes)
else:
names = None
self.current_frame = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
# Create MultiTracker object
multi_tracker = cv2.MultiTracker_create()
# Initialize MultiTracker
for box in detected_boxes:
# box[3] is x,
# box[0] is y,
# box[1] is x + w,
# box[2] is y + h.
reworked_box = box[3], box[
0], box[1] - box[3], box[2] - box[0]
multi_tracker.add(self.create_tracker_by_name(),
self.current_frame, reworked_box)
found_count += 1
# grab the raw NumPy array representing the image, then initialize the timestamp and occupied/unoccupied text
frame = frame.array
self.current_frame = frame.copy(
) # For reaching with overwrite privileges.
if use_detection >= 3:
rgb, detected_boxes = self.detect_things(self.current_frame)
use_detection = 0
use_detection += 1
# Start timer
timer = cv2.getTickCount()
# get updated location of objects in subsequent frames
is_tracking_success, tracked_boxes = multi_tracker.update(
self.current_frame)
if not len(detected_boxes) >= len(tracked_boxes):
d_t_failure_count += 1
else:
d_t_failure_count = 0
# Calculate Frames per second (FPS)
fps = cv2.getTickFrequency() / (cv2.getTickCount() - timer)
if is_tracking_success and d_t_failure_count < 5:
self.track(self.current_frame, tracked_boxes, names)
elif not is_tracking_success or d_t_failure_count >= 5:
# Tracking failure
cv2.putText(self.current_frame, "Tracking failure detected",
(100, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.75,
(0, 0, 255), 2)
tracked_boxes = [] # for clearing tracked_boxes list.
# # Display tracker type on frame
# cv2.putText(frame, self.tracker_type + " Tracker", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
#
# # Display FPS on frame
# cv2.putText(frame, "FPS : " + str(int(fps)), (100, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
self.show_frame(self.current_frame)
self.truncate_stream()
if self.check_loop_ended(stop_thread):
break
def track_without_recognizing(self, frame, boxes, names):
"""The low-level method to track the objects without recognize them, for detect_track methods.
Args:
frame: Frame matrix in rgb format.
boxes: Tuple variable of locations of detected objects.
names: Names of the recognized objects. A person or just an object.
"""
if len(boxes) == 1:
for (x, y, w, h) in boxes:
physically_distance = self.target_locker.get_physically_distance(
w
) # for calculating the just about physically distance of object.
radius = int(sqrt(w * w + h * h) / 2)
self.target_locker.lock(x, y, w, h)
if (self.show_stream and self.augmented) or self.show_stream:
self.mark_object(frame, x, y, w, h, radius,
physically_distance, (255, 0, 0), 2)
# time.sleep(0.1) # Allow the servos to complete the moving.
def track_with_recognizing(self, frame, boxes, names):
"""The low-level method to track the objects with recognize them, for detect_track methods.
Args:
frame: Frame matrix in rgb format.
boxes: Tuple variable of locations of detected objects.
names: Names of the recognized objects. A person or just an object.
"""
for (x, y, w, h), name in zip(boxes, names):
physically_distance = self.target_locker.get_physically_distance(
w
) # for calculating the just about physically distance of object.
radius = int(sqrt(w * w + h * h) / 2)
if name == "Unknown":
if (self.show_stream and self.augmented) or self.show_stream:
frame = self.aimer.mark_rotating_arcs(
frame, (int(x + w / 2), int(y + h / 2)), radius,
physically_distance)
else:
self.target_locker.lock(x, y, w, h)
if (self.show_stream and self.augmented) or self.show_stream:
self.mark_object(frame, x, y, w, h, radius,
physically_distance, (255, 0, 0), 2)
# time.sleep(0.1) # Allow the servos to complete the moving.
def learn(self, stop_thread, format="bgr"):
"""The top-level method to learn how to track objects.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
if self.record:
self.recorder.start("learn")
self.detect_initiate()
for frame in self.camera.capture_continuous(self.rawCapture,
format=format,
use_video_port=True):
# grab the raw NumPy array representing the image, then initialize the timestamp
# and occupied/unoccupied text
image = frame.array
self.current_frame = image.copy(
) # For reaching with overwrite privileges.
# err_check_image = None
rgb, detected_boxes = self.detect_things(self.current_frame)
# names = self.recognize_things(rgb, detected_boxes)
reworked_boxes = self.relocate_detected_coords(detected_boxes)
if not len(reworked_boxes) == 1:
# self.show_frame(self.current_frame)
pass
else:
for (x, y, w, h) in reworked_boxes:
if (self.show_stream
and self.augmented) or self.show_stream:
self.mark_object(self.current_frame, x, y, w, h, 30,
50, (255, 0, 0), 2)
obj_width = w
# obj_area = w * h # unit of obj_width is px ^ 2.
self.target_locker.lock(x, y, w, h)
# time.sleep(0.2) # allow the camera to capture after moving.
# self.show_frame(self.current_frame)
self.truncate_stream()
self.camera.capture(self.rawCapture, format=format)
err_check_image = self.rawCapture.array
rgb, detected_boxes = self.detect_things(err_check_image)
# names = self.recognize_things(rgb, detected_boxes)
rb_after_move = self.relocate_detected_coords(
detected_boxes)
if not len(rb_after_move) == 1:
# self.show_frame(err_check_image)
pass
else:
for (ex, ey, ew,
eh) in rb_after_move: # e means error.
if (self.show_stream
and self.augmented) or self.show_stream:
self.mark_object(self.current_frame, ex, ey,
ew, eh, 30, 50, (255, 0, 0),
2)
self.target_locker.check_error(ex, ey, ew, eh)
# self.show_frame(self.current_frame)
self.show_frame(self.current_frame)
self.truncate_stream()
if self.check_loop_ended(stop_thread):
break
def detect_initiate(self, format="bgr"):
"""The low-level method to serve as the entry point to detection, recognition and tracking features of t_system's vision ability.
Args:
format: Color space format.
"""
detected_boxes = []
rgb = None
for frame in self.camera.capture_continuous(self.rawCapture,
format=format,
use_video_port=True):
frame = frame.array
self.current_frame = frame.copy(
) # For reaching to frame with overwrite privileges.
rgb, detected_boxes = self.detect_things(self.current_frame)
if not detected_boxes:
gray = cv2.cvtColor(rgb, cv2.COLOR_BGR2GRAY)
if (self.show_stream and self.augmented) or self.show_stream:
cv2.putText(
gray, "Scanning...",
(int(self.frame_width - self.frame_width * 0.2),
int(self.frame_height * 0.1)),
cv2.FONT_HERSHEY_SIMPLEX, 0.75, (200, 0, 0), 2)
self.show_frame(gray)
self.truncate_stream()
if self.check_loop_ended(lambda: False):
break
else:
self.rawCapture.truncate(0)
break
return rgb, detected_boxes
def security(self, stop_thread, format="bgr"):
"""The top-level method to provide the security via scanning and taking photos.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
import threading
global thread_of_scan
global thread_of_stream
is_first_run = True
while True:
if is_first_run or stop_thread():
if not stop_thread():
thread_of_scan = threading.Thread(target=self.scan,
args=(stop_thread, 3))
thread_of_stream = threading.Thread(target=self.stream,
args=(stop_thread,
format,
"security"))
thread_of_scan.start()
thread_of_stream.start()
else:
# print("thread killed")
thread_of_scan.join()
thread_of_stream.join()
break
is_first_run = False
time.sleep(0.5) # for relieve the cpu.
def scan(self, stop_thread, resolution=3):
"""The low-level method to provide the scanning around for security mode of T_System.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
resolution: angle's step width between 0 - 180 degree.
"""
while True:
if stop_thread():
break
for angle in range(0, 180, resolution):
if stop_thread():
break
self.target_locker.pan.move(float(angle), 180.0)
angle_for_ellipse_move = calc_ellipsoidal_angle(
float(angle) - 90, 180.0,
75.0) # 75 degree is for physical conditions.
self.target_locker.tilt.move(
angle_for_ellipse_move,
75.0) # last parameter not used for both funcs
time.sleep(0.1)
for angle in range(180, 0, resolution * -1):
if stop_thread():
break
self.target_locker.pan.move(float(angle), 180.0)
angle_for_ellipse_move = calc_ellipsoidal_angle(
float(angle) - 90, 180.0, 75.0)
self.target_locker.tilt.move(
angle_for_ellipse_move,
75.0) # last parameter not used for both funcs
time.sleep(0.1)
def stream(self, stop_thread, format="bgr", caller="security"):
"""The low-level method to provide the video stream for security mode of T_System.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format (str): Color space format.
caller (str): The method that calls the stream.
"""
if self.record:
self.recorder.start(caller)
for frame in self.camera.capture_continuous(self.rawCapture,
format=format,
use_video_port=True):
# inside of the loop is optionally editable.
self.current_frame = frame.array
self.show_frame(self.current_frame)
self.truncate_stream()
if self.check_loop_ended(stop_thread):
break
def track_focused_point(self):
"""The high-level method to provide the tracking predetermined non-moving target according to with locking_system's current position for track mode of T_System.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
pass
def show_frame(self, frame):
"""The low-level method to show the captured frame.
Args:
frame: Frame matrix in bgr format.
"""
if (self.show_stream and self.augmented) or self.show_stream:
# show the frame
cv2.imshow("Frame", frame)
def truncate_stream(self, ):
"""The low-level method to clear the stream in preparation for the next frame.
"""
self.rawCapture.truncate(0)
def check_loop_ended(self, stop_thread):
"""The low-level method to detecting FACES with hog or cnn methoda.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
"""
# if the `q` key was pressed, break from the loop
key = cv2.waitKey(1) & 0xFF
if (key == ord("q") or stop_thread()) and self.augmented:
# print("thread killed")
return True
elif (key == ord("q") or stop_thread()) and not self.augmented:
cv2.destroyAllWindows()
self.release_camera()
self.release_servos()
self.release_hearer()
return True
def detect_with_hog_or_cnn(self, frame):
"""The low-level method to detecting FACES with hog or cnn methoda.
Args:
frame: Frame matrix in bgr format.
"""
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# rgb = cv2.resize(rgb, (0, 0), fx=0.25, fy=0.25)
# r = frame.shape[1] / float(rgb.shape[1])
detected_boxes = face_recognition.face_locations(
rgb, model=self.detection_model)
return rgb, detected_boxes
def detect_with_haarcascade(self, frame):
"""The low-level method to detecting objects with haarcascade method.
Args:
frame: Frame matrix in bgr format.
"""
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# gray = cv2.equalizeHist(gray)
detected_boxes = self.object_cascade.detectMultiScale(gray, 1.3, 5)
# Following list's member change is for face recognition format compatibility.
reworked_boxes = []
for box in detected_boxes:
# box[3] is x,
# box[0] is y,
# box[1] is x + w,
# box[2] is y + h.
reworked_boxes.append(
(box[1], box[0] + box[2], box[1] + box[3], box[0]))
rgb = cv2.cvtColor(
frame, cv2.COLOR_BGR2RGB) # For recognize_things compatibility.
return rgb, reworked_boxes
def recognize_things(self, rgb, boxes):
"""The low-level method to recognizing objects with encodings pickle files.
Args:
rgb: Frame matrix in rgb format.
boxes: Tuple variable of locations of detected objects.
"""
encodings = face_recognition.face_encodings(rgb, boxes)
names = []
for encoding in encodings:
# attempt to match each face in the input image to our known encodings
matches = face_recognition.compare_faces(
self.recognition_data["encodings"], encoding)
name = "Unknown"
# check to see if we have found a match
if True in matches:
# find the indexes of all matched faces then initialize a
# dictionary to count the total number of times each face
# was matched
matched_idxs = [i for (i, b) in enumerate(matches) if b]
counts = {}
# loop over the matched indexes and maintain a count for
# each recognized face
for i in matched_idxs:
name = self.recognition_data["names"][i]
counts[name] = counts.get(name, 0) + 1
# determine the recognized face with the largest number
# of votes (note: in the event of an unlikely tie Python
# will select first entry in the dictionary)
name = max(counts, key=counts.get)
# update the list of names
names.append(name)
return names
def create_tracker_by_name(self):
"""The low-level method to creating a tracker object via type that is chosen by the user.
"""
if self.tracker_type == TRACKER_TYPES[0]:
tracker = cv2.TrackerBoosting_create()
elif self.tracker_type == TRACKER_TYPES[1]:
tracker = cv2.TrackerMIL_create()
elif self.tracker_type == TRACKER_TYPES[2]:
tracker = cv2.TrackerKCF_create()
elif self.tracker_type == TRACKER_TYPES[3]:
tracker = cv2.TrackerTLD_create()
elif self.tracker_type == TRACKER_TYPES[4]:
tracker = cv2.TrackerMedianFlow_create()
elif self.tracker_type == TRACKER_TYPES[5]:
tracker = cv2.TrackerGOTURN_create()
elif self.tracker_type == TRACKER_TYPES[6]:
tracker = cv2.TrackerMOSSE_create()
elif self.tracker_type == TRACKER_TYPES[7]:
tracker = cv2.TrackerCSRT_create()
else:
tracker = None
print('Incorrect tracker name')
print('Available trackers are:')
for t in TRACKER_TYPES:
print(t)
return tracker
@staticmethod
def relocate_detected_coords(detected_boxes):
"""The low-level method to relocating members of detected boxes from given shape to wanted shape.
Args:
detected_boxes (tuple): Top left and bottom right coordinates of the detected thing.
"""
reworked_boxes = []
for box in detected_boxes:
# box[3] is x,
# box[0] is y,
# box[1] is x + w,
# box[2] is y + h.
reworked_boxes.append(
(box[3], box[0], box[1] - box[3], box[2] - box[0]))
return reworked_boxes
def change_tracked_thing(self, file):
"""The top-level method to changing tracking objects.
Args:
file: The haarcascade trained xml file.
"""
ccade_xml_file = T_SYSTEM_PATH + "/haarcascade/" + file + ".xml"
self.object_cascade = cv2.CascadeClassifier(ccade_xml_file)
self.decider.set_db(file)
def mark_as_single_rect(self, frame, x, y, w, h, radius,
physically_distance, color, thickness):
"""The low-level method to set mark_object method as drawing method with OpenCV's basic rectangle.
Args:
frame: Frame matrix.
x : the column number of the top left point of found object from the detection method.
y : the row number of the top left point of found object from the detection method.
w : the width of found object from the detection method.
h : the height of found object from the detection method.
radius: Radius of the aim.
physically_distance: Physically distance of the targeted object as pixel count.
color (tuple): Color of the drawing shape. In RGB Space.
thickness (int): Thickness of the drawing shape.
"""
cv2.rectangle(frame, (x, y), (x + w, y + h), color, thickness)
def mark_as_partial_rect(self, frame, x, y, w, h, radius,
physically_distance, color, thickness):
"""The low-level method to set mark_object method as drawing method with aimer's partial rect.
Args:
frame: Frame matrix.
x : the column number of the top left point of found object from the detection method.
y : the row number of the top left point of found object from the detection method.
w : the width of found object from the detection method.
h : the height of found object from the detection method.
radius: Radius of the aim.
physically_distance: Physically distance of the targeted object as pixel count.
color (tuple): Color of the drawing shape. In RGB Space.
thickness (int): Thickness of the drawing shape.
"""
self.aimer.mark_partial_rect(frame, (int(x + w / 2), int(y + h / 2)),
radius, physically_distance)
def mark_as_rotation_arcs(self, frame, x, y, w, h, radius,
physically_distance, color, thickness):
"""The low-level method to set mark_object method as drawing method with aimer's rotating arcs.
Args:
frame: Frame matrix.
x : the column number of the top left point of found object from the detection method.
y : the row number of the top left point of found object from the detection method.
w : the width of found object from the detection method.
h : the height of found object from the detection method.
radius: Radius of the aim.
physically_distance: Physically distance of the targeted object as pixel count.
color (tuple): Color of the drawing shape. In RGB Space.
thickness (int): Thickness of the drawing shape.
"""
self.aimer.mark_rotating_arcs(frame, (int(x + w / 2), int(y + h / 2)),
radius, physically_distance)
def mark_as_none(self, frame, x, y, w, h, radius, physically_distance,
color, thickness):
"""The low-level method to set mark_object method for draw nothing.
Args:
frame: Frame matrix.
x : the column number of the top left point of found object from the detection method.
y : the row number of the top left point of found object from the detection method.
w : the width of found object from the detection method.
h : the height of found object from the detection method.
radius: Radius of the aim.
physically_distance: Physically distance of the targeted object as pixel count.
color (tuple): Color of the drawing shape. In RGB Space.
thickness (int): Thickness of the drawing shape.
"""
pass
def get_mark_object(self, mark_found_object):
"""The low-level method to set mqtt_receimitter object for publishing and subscribing data echos.
Args:
mark_found_object (str): The mark type of the detected object.
"""
if mark_found_object == "single_rect":
return self.mark_as_single_rect
elif mark_found_object == "partial_rect":
return self.mark_as_partial_rect
elif mark_found_object == "rotating_arcs":
return self.mark_as_rotation_arcs
else:
return self.mark_as_none
def get_current_frame(self):
"""The high-level method to get current working camera frame.
"""
return self.current_frame
def set_mqtt_receimitter(self, mqtt_receimitter):
"""The low-level method to set mqtt_receimitter object for publishing and subscribing data echos.
Args:
mqtt_receimitter: transmit and receive data function for mqtt communication
"""
self.mqtt_receimitter = mqtt_receimitter
def start_preview(self):
"""The high-level method to start preview of the vision without move action.
"""
self.camera.start_preview()
def stop_preview(self):
"""The high-level method to stop preview of the vision.
"""
self.camera.stop_preview()
def release_servos(self):
"""The low-level method to stop sending signals to servo motors pins and clean up the gpio pins.
"""
self.target_locker.stop()
self.target_locker.gpio_cleanup()
def release_camera(self):
"""The low-level method to stop receiving signals from the camera and stop video recording.
"""
# self.camera.release()
if self.record:
self.camera.stop_recording()
def release_hearer(self):
"""The low-level method to stop sending signals to servo motors pins and clean up the gpio pins.
"""
self.hearer.release_members()
class Vision():
"""Class to define a vision of tracking system..
This class provides necessary initiations and functios named :func:`t_system.vision.Vision.detect_track`
as the loop for each camera frames for tracking mode, named :func:`t_system.vision.Vision.learn` as the
learning ability and :func:`t_system.vision.Vision.security` as the security mode.
"""
def __init__(self, args, camera, resolution=(320, 240), framerate=32):
"""Initialization method of :class:`t_system.vision.Vision` class.
Args:
camera: Camera object from PiCamera.
resolution: rPi camera's resolution data.
framerate: rPi camera's framerate data.
"""
if not args["cascadefile"]:
ccade_xml_file = T_SYSTEM_PATH + "/haarcascade/frontalface_default.xml"
self.decider = Decider()
else:
ccade_xml_file = T_SYSTEM_PATH + "/haarcascade/" + args["cascadefile"] + ".xml"
self.decider = Decider(args["cascadefile"])
self.servo_pan = Motor(17, self.decider) # pan means rotate right and left ways.
self.servo_tilt = Motor(14, self.decider, 4.5, False) # tilt means rotate up and down ways.
self.aimer = Aimer()
self.show_stream = args["show_stream"] # 'show-stream' argument automatically converted this type.
self.augmented = args["augmented"]
self.camera = camera
self.camera.resolution = resolution
self.camera.framerate = framerate
# self.camera.start_preview()
self.rawCapture = PiRGBArray(camera, size=resolution)
self.object_cascade = cv2.CascadeClassifier(ccade_xml_file)
(self.frame_width, self.frame_height) = resolution
self.mqtt_receimitter = None
# allow the camera to warmup
time.sleep(0.1)
def detect_track(self, stop_thread, format="bgr"):
"""The top-level method to real time object detection.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
for frame in self.camera.capture_continuous(self.rawCapture, format=format, use_video_port=True):
# grab the raw NumPy array representing the image, then initialize the timestamp
# and occupied/unoccupied text
image = frame.array
editable_image = image.copy()
gray = cv2.cvtColor(editable_image, cv2.COLOR_BGR2GRAY)
# gray = cv2.equalizeHist(gray)
objects = self.object_cascade.detectMultiScale(gray, 1.3, 5)
if not len(objects) == 1:
pass
else:
for (x, y, w, h) in objects:
if self.show_stream or not self.augmented:
radius = int(sqrt(w * w + h * h) / 2)
# cv2.rectangle(editable_image, (x, y), (x + w, y + h), (255, 0, 0), 2)
# cv2.circle(editable_image, (int(x + w / 2), int(y + h / 2)), radius, (0, 255, 0), 1)
obj_width_pan, physically_distance = self.servo_pan.move(x, x + w, self.frame_width)
self.servo_tilt.move(y, y + h, self.frame_height)
editable_image = self.aimer.mark(editable_image, (int(x + w / 2), int(y + h / 2)), radius, physically_distance)
time.sleep(0.1) # allow the servos to complete the moving.
if self.show_stream or not self.augmented:
# show the frame
cv2.imshow("Frame", editable_image)
key = cv2.waitKey(1) & 0xFF
# clear the stream in preparation for the next frame
self.rawCapture.truncate(0)
# if the `q` key was pressed, break from the loop
if key == ord("q") or stop_thread():
# print("thread killed")
break
def learn(self, stop_thread, format="bgr"):
"""The top-level method to learn how to track objects.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
for frame in self.camera.capture_continuous(self.rawCapture, format=format, use_video_port=True):
# grab the raw NumPy array representing the image, then initialize the timestamp
# and occupied/unoccupied text
image = frame.array
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# gray = cv2.equalizeHist(gray)
objects = self.object_cascade.detectMultiScale(gray, 1.3, 5)
if not len(objects) == 1:
pass
else:
for (x, y, w, h) in objects:
if self.show_stream or not self.augmented:
cv2.rectangle(image, (x, y), (x + w, y + h), (255, 0, 0), 2)
obj_width_pan, physically_distance = self.servo_pan.move(x, x + w, self.frame_width)
obj_width_tilt, physically_distance = self.servo_tilt.move(y, y + h, self.frame_height)
time.sleep(0.5) # allow the camera to capture after moving.
self.rawCapture.truncate(0) # for emptying arrays for capturing frame again.
self.camera.capture(self.rawCapture, format=format)
error_check_image = self.rawCapture.array
error_check_gray = cv2.cvtColor(error_check_image, cv2.COLOR_BGR2GRAY)
# gray = cv2.equalizeHist(gray)
objects_after_move = self.object_cascade.detectMultiScale(error_check_gray, 1.3, 5)
if not len(objects_after_move) == 1:
pass
else:
for (x, y, w, h) in objects_after_move:
if self.show_stream or not self.augmented:
cv2.rectangle(image, (x, y), (x + w, y + h), (255, 0, 0), 2)
err_rate_pan = float(self.servo_pan.current_dis_to_des(x, x+w, self.frame_width) / self.servo_pan.get_previous_dis_to_des()) * 100
err_rate_tilt = float(self.servo_tilt.current_dis_to_des(y, y+h, self.frame_height) / self.servo_tilt.get_previous_dis_to_des()) * 100
self.decider.decision(obj_width_pan, err_rate_pan, True)
self.decider.decision(obj_width_tilt, err_rate_tilt, True)
if self.show_stream or not self.augmented:
# show the frame
cv2.imshow("Frame", image)
key = cv2.waitKey(1) & 0xFF
# clear the stream in preparation for the next frame
self.rawCapture.truncate(0)
# if the `q` key was pressed, break from the loop
if key == ord("q") or stop_thread():
# print("thread killed")
break
def security(self, stop_thread, format="bgr"):
"""The top-level method to provide the security via scanning and taking photos.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
import threading
global thread_of_scan
global thread_of_stream
is_first_run = True
while True:
if is_first_run or stop_thread():
if not stop_thread():
thread_of_scan = threading.Thread(target=self.scan, args=(stop_thread, 3))
thread_of_stream = threading.Thread(target=self.stream, args=(stop_thread, format))
thread_of_scan.start()
thread_of_stream.start()
else:
# print("thread killed")
thread_of_scan.join()
thread_of_stream.join()
break
is_first_run = False
time.sleep(0.5) # for relieve the cpu.
def scan(self, stop_thread, resolution=3):
"""The top-level method to provide the scanning around for security mode of T_System.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
resolution: angle's step width between 0 - 180 degree.
"""
while True:
if stop_thread():
break
for angle in range(0, 180, resolution):
if stop_thread():
break
self.servo_pan.angular_move(float(angle), 180.0)
angle_for_ellipse_move = calc_ellipsoidal_angle(float(angle) - 90, 180.0, 75.0) # 75 degree is for physical conditions.
self.servo_tilt.angular_move(angle_for_ellipse_move, 75.0)
time.sleep(0.1)
for angle in range(180, 0, resolution * -1):
if stop_thread():
break
self.servo_pan.angular_move(float(angle), 180.0)
angle_for_ellipse_move = calc_ellipsoidal_angle(float(angle) - 90, 180.0, 75.0)
self.servo_tilt.angular_move(angle_for_ellipse_move, 75.0)
time.sleep(0.1)
def stream(self, stop_thread, format="bgr"):
"""The top-level method to provide the video stream for security mode of T_System.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
resolution: angle's step width between 0 - 180 degree.
"""
for frame in self.camera.capture_continuous(self.rawCapture, format=format, use_video_port=True):
# grab the raw NumPy array representing the image, then initialize the timestamp
# and occupied/unoccupied text
image = frame.array
# gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# gray = cv2.equalizeHist(gray)
# SECURITY MODE CAN BE TAKING PHOTOGRAPHS AS DIFFERENT FROM learn AND detect_track FUNCTIONS.
# objects = self.object_cascade.detectMultiScale(gray, 1.3, 5)
#
# if not len(objects) == 1:
# pass
# else:
# for (x, y, w, h) in objects:
# if self.show_stream or not self.augmented:
# cv2.rectangle(image, (x, y), (x + w, y + h), (255, 0, 0), 2)
if self.show_stream or not self.augmented:
# show the frame
cv2.imshow("Frame", image)
key = cv2.waitKey(1) & 0xFF
# clear the stream in preparation for the next frame
self.rawCapture.truncate(0)
# if the `q` key was pressed, break from the loop
if key == ord("q") or stop_thread():
break
# if loop end, the scan process will be terminated.
def change_tracked_thing(self, file):
"""The top-level method to changing tracking objects.
Args:
file: The haarcascade trained xml file.
"""
ccade_xml_file = T_SYSTEM_PATH + "/" + file
self.object_cascade = cv2.CascadeClassifier(ccade_xml_file)
self.decider.set_db(file)
def set_mqtt_receimitter(self, mqtt_receimitter):
"""The top-level method to set mqtt_receimitter object for publishing and subscribing data echos.
Args:
mqtt_receimitter: transmit and receive data function for mqtt communication
"""
self.mqtt_receimitter = mqtt_receimitter
def __init__(self, args):
"""Initialization method of :class:`t_system.vision.Vision` class.
Args:
args: Command-line arguments.
"""
self.config_file = f'{T_SYSTEM_PATH}/vision/vision_config.json'
with open(self.config_file) as conf_file:
conf_file_json = json.load(conf_file)
self.tracker_types = conf_file_json[
"tracker_types"] # config file returns the tracker type list.
self.target_mark_types = conf_file_json[
"target_mark_types"] # config file returns the mark type dict.
self.detection_model = args["detection_model"]
if self.detection_model == "haarcascade":
self.detect_things = self.__detect_with_haarcascade
else:
self.detect_things = self.__detect_with_hog_or_cnn
if args['use_tracking_api']:
self.detect_track = self.__d_t_with_cv_ta
else:
self.detect_track = self.__d_t_without_cv_ta
# Specify the tracker type
self.tracker_type = args["tracker_type"]
self.no_recognize = args["no_recognize"]
if not self.no_recognize:
self.recognition_data = self.__get_recognition_data(
args["encoding_file"])
self.track = self.__track_with_recognizing
else:
self.track = self.__track_without_recognizing
self.hearer = Hearer(args)
# self.hearer = None
resolution = (args["resolution"][0], args["resolution"][1])
self.camera = PiCamera()
self.camera.resolution = resolution
self.camera.framerate = args["framerate"]
self.camera.rotation = args["camera_rotation"]
self.recorder = Recorder(args["shot_format"], args["shoot_formats"],
self.camera, self.hearer)
self.online_streamer = OnlineStreamer(self.camera, self.hearer)
self.raw_capture = PiRGBArray(self.camera, size=resolution)
self.object_cascades = self.set_object_cascades(args["cascades"])
(self.frame_width, self.frame_height) = resolution
self.decider = None
if args["AI"] == "official_ai":
self.decider = Decider(args["cascades"][0])
self.target_locker = LockingSystem(args, resolution, self.decider)
self.aimer = Aimer()
self.show_stream = args[
"show_stream"] # 'show-stream' argument automatically converted this type.
self.mark_object = self.__get_mark_object(args["found_object_mark"])
self.record = args["record"]
self.augmented = False
if args["interface"] == "augmented":
self.augmented = True
self.mqtt_receimitter = None
self.current_frame = np.zeros(shape=(self.frame_height,
self.frame_width))
self.stop_thread = False
self.active_threads = []
self.is_watching = False
self.obj_detected = False
class Vision:
"""Class to define a vision of tracking system..
This class provides necessary initiations and functions named :func:`t_system.vision.Vision.detect_track`
as the loop for each camera frames for tracking mode, named :func:`t_system.vision.Vision.learn` as the
learning ability and :func:`t_system.vision.Vision.security` as the security mode.
"""
def __init__(self, args):
"""Initialization method of :class:`t_system.vision.Vision` class.
Args:
args: Command-line arguments.
"""
self.config_file = f'{T_SYSTEM_PATH}/vision/vision_config.json'
with open(self.config_file) as conf_file:
conf_file_json = json.load(conf_file)
self.tracker_types = conf_file_json[
"tracker_types"] # config file returns the tracker type list.
self.target_mark_types = conf_file_json[
"target_mark_types"] # config file returns the mark type dict.
self.detection_model = args["detection_model"]
if self.detection_model == "haarcascade":
self.detect_things = self.__detect_with_haarcascade
else:
self.detect_things = self.__detect_with_hog_or_cnn
if args['use_tracking_api']:
self.detect_track = self.__d_t_with_cv_ta
else:
self.detect_track = self.__d_t_without_cv_ta
# Specify the tracker type
self.tracker_type = args["tracker_type"]
self.no_recognize = args["no_recognize"]
if not self.no_recognize:
self.recognition_data = self.__get_recognition_data(
args["encoding_file"])
self.track = self.__track_with_recognizing
else:
self.track = self.__track_without_recognizing
self.hearer = Hearer(args)
# self.hearer = None
resolution = (args["resolution"][0], args["resolution"][1])
self.camera = PiCamera()
self.camera.resolution = resolution
self.camera.framerate = args["framerate"]
self.camera.rotation = args["camera_rotation"]
self.recorder = Recorder(args["shot_format"], args["shoot_formats"],
self.camera, self.hearer)
self.online_streamer = OnlineStreamer(self.camera, self.hearer)
self.raw_capture = PiRGBArray(self.camera, size=resolution)
self.object_cascades = self.set_object_cascades(args["cascades"])
(self.frame_width, self.frame_height) = resolution
self.decider = None
if args["AI"] == "official_ai":
self.decider = Decider(args["cascades"][0])
self.target_locker = LockingSystem(args, resolution, self.decider)
self.aimer = Aimer()
self.show_stream = args[
"show_stream"] # 'show-stream' argument automatically converted this type.
self.mark_object = self.__get_mark_object(args["found_object_mark"])
self.record = args["record"]
self.augmented = False
if args["interface"] == "augmented":
self.augmented = True
self.mqtt_receimitter = None
self.current_frame = np.zeros(shape=(self.frame_height,
self.frame_width))
self.stop_thread = False
self.active_threads = []
self.is_watching = False
self.obj_detected = False
# Allow the camera to warm up
# time.sleep(0.1)
def watch(self, stop_thread, format="bgr", caller="security"):
"""The top-level method to provide the video stream for security mode of T_System.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format (str): Color space format.
caller (str): The method that calls the stream.
"""
self.is_watching = True
if self.record:
self.recorder.start_shoot(caller)
logger.debug("stream starting with capture_continuous")
for frame in self.camera.capture_continuous(self.raw_capture,
format=format,
use_video_port=True):
if not self.obj_detected:
self.current_frame = frame.array.copy()
self.__show_frame(self.current_frame)
self.__truncate_stream()
if self.__check_loop_ended(stop_thread):
break
else:
self.__truncate_stream()
time.sleep(0.1)
if self.record:
self.stop_recording()
self.is_watching = False
def watch_and(self, task):
"""Method to provide starting watching ability of the around of Vision and accordingly starting given task.
Args:
task: Task for the seer. Either `learn`, `track` or `secure`.
"""
if task == "learn":
learn_thread = threading.Thread(target=self.learn,
args=(lambda: self.stop_thread, ))
self.active_threads.append(learn_thread)
learn_thread.start()
elif task == "track":
track_thread = threading.Thread(target=self.detect_track,
args=(lambda: self.stop_thread, ))
self.active_threads.append(track_thread)
track_thread.start()
elif task == "secure":
secure_thread = threading.Thread(target=self.scan,
args=(lambda: self.stop_thread,
3))
secure_thread.start()
self.active_threads.append(secure_thread)
def __d_t_without_cv_ta(self, stop_thread, format='bgr'):
"""Method to provide detecting and tracking objects without using OpenCV's tracking API.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
self.is_watching = True
self.start_recording("track")
for frame in self.camera.capture_continuous(self.raw_capture,
format=format,
use_video_port=True):
if not self.obj_detected:
self.current_frame = frame.array.copy()
self.__truncate_stream()
rgb, detected_boxes = self.detect_things(self.current_frame)
if detected_boxes:
self.obj_detected = True
reworked_boxes = self.__relocate_detected_coords(
detected_boxes)
if not self.no_recognize:
names = self.__recognize_things(rgb, detected_boxes)
else:
names = None
self.track(self.current_frame, reworked_boxes, names)
self.obj_detected = False
if self.__check_loop_ended(stop_thread):
break
self.stop_recording()
self.is_watching = False
def __d_t_with_cv_ta(self, stop_thread, format='bgr'):
"""Method to provide detecting and tracking objects with using OpenCV's tracking API.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
self.is_watching = True
self.start_recording("track")
tracked_boxes = [] # this became array. because of overriding.
names = []
multi_tracker = cv2.MultiTracker_create()
rgb, detected_boxes = self.detect_initiate(stop_thread)
found_count = 0
d_t_failure_count = 0
use_detection = 0
for frame in self.camera.capture_continuous(self.raw_capture,
format=format,
use_video_port=True):
self.current_frame = frame.array.copy()
self.__truncate_stream()
if len(detected_boxes) > len(tracked_boxes):
if not self.no_recognize:
names = self.__recognize_things(rgb, detected_boxes)
else:
names = None
self.current_frame = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
# Create MultiTracker object
multi_tracker = cv2.MultiTracker_create()
# Initialize MultiTracker
for box in detected_boxes:
# box[3] is x,
# box[0] is y,
# box[1] is x + w,
# box[2] is y + h.
reworked_box = box[3], box[
0], box[1] - box[3], box[2] - box[0]
multi_tracker.add(self.__create_tracker_by_name(),
self.current_frame, reworked_box)
found_count += 1
if use_detection >= 3:
rgb, detected_boxes = self.detect_things(self.current_frame)
use_detection = 0
use_detection += 1
# Start timer
timer = cv2.getTickCount()
# get updated location of objects in subsequent frames
is_tracking_success, tracked_boxes = multi_tracker.update(
self.current_frame)
if not len(detected_boxes) >= len(tracked_boxes):
d_t_failure_count += 1
else:
d_t_failure_count = 0
# Calculate Frames per second (FPS)
fps = cv2.getTickFrequency() / (cv2.getTickCount() - timer)
if is_tracking_success and d_t_failure_count < 5:
self.track(self.current_frame, tracked_boxes, names)
elif not is_tracking_success or d_t_failure_count >= 5:
# Tracking failure
cv2.putText(self.current_frame, "Tracking failure detected",
(100, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.75,
(0, 0, 255), 2)
tracked_boxes = [] # for clearing tracked_boxes list.
# # Display tracker type on frame
# cv2.putText(self.current_frame, self.tracker_type + " Tracker", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
#
# # Display FPS on frame
# cv2.putText(self.current_frame, "FPS : " + str(int(fps)), (100, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
if self.__check_loop_ended(stop_thread):
break
self.stop_recording()
self.is_watching = False
def __track_without_recognizing(self, frame, boxes, names):
"""Method to track the objects without recognize them, for detect_track methods.
Args:
frame: Frame matrix in rgb format.
boxes: Tuple variable of locations of detected objects.
names: Names of the recognized objects. A person or just an object.
"""
if len(boxes) == 1:
for (x, y, w, h) in boxes:
physically_distance = self.target_locker.get_physically_distance(
w
) # for calculating the just about physically distance of object.
radius = int(sqrt(w * w + h * h) / 2)
self.target_locker.lock(x, y, w, h)
self.mark_object(frame, x, y, w, h, radius,
physically_distance, (255, 0, 0), 2)
if (self.show_stream and self.augmented) or self.show_stream:
self.__show_frame(frame)
# time.__sleep(0.1) # Allow the servos to complete the moving.
def __track_with_recognizing(self, frame, boxes, names):
"""Method to track the objects with recognize them, for detect_track methods.
Args:
frame: Frame matrix in rgb format.
boxes: Tuple variable of locations of detected objects.
names: Names of the recognized objects. A person or just an object.
"""
for (x, y, w, h), name in zip(boxes, names):
physically_distance = self.target_locker.get_physically_distance(
w
) # for calculating the just about physically distance of object.
radius = int(sqrt(w * w + h * h) / 2)
if name == "Unknown":
if (self.show_stream and self.augmented) or self.show_stream:
frame = self.aimer.mark_rotating_arcs(
frame, (int(x + w / 2), int(y + h / 2)), radius,
physically_distance)
else:
self.target_locker.lock(x, y, w, h)
self.mark_object(frame, x, y, w, h, radius,
physically_distance, (255, 0, 0), 2)
# time.__sleep(0.1) # Allow the servos to complete the moving.
def learn(self, stop_thread, format="bgr"):
"""The top-level method to learn how to track objects.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
self.detect_initiate(stop_thread)
self.is_watching = True
self.start_recording("learn")
for frame in self.camera.capture_continuous(self.raw_capture,
format=format,
use_video_port=True):
self.current_frame = frame.array.copy()
self.__truncate_stream()
rgb, detected_boxes = self.detect_things(self.current_frame)
# names = self.__recognize_things(rgb, detected_boxes)
reworked_boxes = self.__relocate_detected_coords(detected_boxes)
if not len(reworked_boxes) == 1:
# self.__show_frame(self.current_frame)
pass
else:
for (x, y, w, h) in reworked_boxes:
if (self.show_stream
and self.augmented) or self.show_stream:
self.mark_object(self.current_frame, x, y, w, h, 30,
50, (255, 0, 0), 2)
obj_width = w
# obj_area = w * h # unit of obj_width is px ^ 2.
self.target_locker.lock(x, y, w, h)
# time.__sleep(0.2) # allow the camera to capture after moving.
for new_frame in self.camera.capture_continuous(
self.raw_capture, format=format,
use_video_port=True):
self.current_frame = new_frame.array.copy()
self.__truncate_stream()
rgb, detected_boxes = self.detect_things(
self.current_frame)
# names = self.__recognize_things(rgb, detected_boxes)
rb_after_move = self.__relocate_detected_coords(
detected_boxes)
if not len(rb_after_move) == 1:
pass
else:
for (ex, ey, ew,
eh) in rb_after_move: # e means error.
if (self.show_stream and
self.augmented) or self.show_stream:
self.mark_object(self.current_frame, ex,
ey, ew, eh, 30, 50,
(255, 0, 0), 2)
self.target_locker.check_error(ex, ey, ew, eh)
break
if self.__check_loop_ended(stop_thread):
break
self.stop_recording()
self.is_watching = False
def detect_initiate(self, stop_thread, format="bgr"):
"""Method to serve as the entry point to detection, recognition and tracking features of t_system's vision ability.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
format: Color space format.
"""
self.is_watching = True
detected_boxes = []
rgb = None
self.target_locker.scan(False)
for frame in self.camera.capture_continuous(self.raw_capture,
format=format,
use_video_port=True):
self.current_frame = frame.array
self.__truncate_stream()
rgb, detected_boxes = self.detect_things(self.current_frame)
if not detected_boxes:
gray = cv2.cvtColor(rgb, cv2.COLOR_BGR2GRAY)
if (self.show_stream and self.augmented) or self.show_stream:
cv2.putText(
gray, "Scanning...",
(int(self.frame_width - self.frame_width * 0.2),
int(self.frame_height * 0.1)),
cv2.FONT_HERSHEY_SIMPLEX, 0.75, (200, 0, 0), 2)
# self.__show_frame(gray, "Initiate")
if self.__check_loop_ended(lambda: False):
break
else:
self.target_locker.scan_thread_stop = True
break
if self.__check_loop_ended(lambda: False) or self.stop_thread:
self.target_locker.scan_thread_stop = True
break
self.is_watching = False
return rgb, detected_boxes
def scan(self, stop_thread, resolution=3):
"""Method to provide the scanning around for security mode of T_System.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
resolution: angle's step width between 0 - 180 degree.
"""
while True:
if stop_thread():
break
for angle in range(0, 180, resolution):
if stop_thread():
break
self.target_locker.pan.move(float(angle), 180.0)
angle_for_ellipse_move = calc_ellipsoidal_angle(
float(angle) - 90, 180.0,
75.0) # 75 degree is for physical conditions.
self.target_locker.tilt.move(
angle_for_ellipse_move,
75.0) # last parameter not used for both funcs
time.sleep(0.1)
for angle in range(180, 0, resolution * -1):
if stop_thread():
break
self.target_locker.pan.move(float(angle), 180.0)
angle_for_ellipse_move = calc_ellipsoidal_angle(
float(angle) - 90, 180.0, 75.0)
self.target_locker.tilt.move(
angle_for_ellipse_move,
75.0) # last parameter not used for both funcs
time.sleep(0.1)
def reload_target_locker(self,
ai=None,
non_moving_target=None,
arm_expansion=None,
current_angles=None):
"""The top-level method to set locking system's locker of Vision as given AI and target object status parameters.
Args:
ai (str): AI type that will using during locking the target.
non_moving_target (bool): Non-moving target flag.
arm_expansion (bool): Flag for the loading locker as expansion of the T_System's robotic arm.
current_angles (list): Current angles of the target locking system's collimators.
"""
if current_angles is None:
current_angles = [None, None]
return self.target_locker.load_locker(ai, non_moving_target,
arm_expansion, current_angles)
def serve_frame_online(self):
"""The top-level method to provide the serving video stream online for sending Flask framework based remote_ui.
"""
is_success, im_buf_arr = cv2.imencode(".jpg", self.current_frame)
return im_buf_arr.tobytes() # image in byte format.
def track_focused_point(self):
"""Method to provide the tracking predetermined non-moving target according to with locking_system's current position for track mode of T_System.
"""
pass
def __show_frame(self, frame, window="Frame"):
"""Method to show the captured frame.
Args:
frame: Frame matrix in bgr format.
window: Name of the frame showing window.
"""
if (self.show_stream and self.augmented) or self.show_stream:
# show the frame
cv2.imshow(window, frame)
def __truncate_stream(self):
"""Method to clear the stream in preparation for the next frame.
"""
self.raw_capture.seek(0)
self.raw_capture.truncate()
def __check_loop_ended(self, stop_thread):
"""Method to control end of the vision loop.
Args:
stop_thread: Stop flag of the tread about terminating it outside of the function's loop.
"""
# if the `q` key was pressed, break from the loop
key = cv2.waitKey(1) & 0xFF
if (key == ord("q") or stop_thread()) and (self.augmented
or self.active_threads):
logger.info("All threads killed. In progress...")
return True
elif (key == ord("q") or stop_thread()) and not self.augmented:
cv2.destroyAllWindows()
self.release_members()
return True
def __detect_with_hog_or_cnn(self, frame):
"""Method to detecting FACES with hog or cnn methoda.
Args:
frame: Frame matrix in bgr format.
"""
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# rgb = cv2.resize(rgb, (0, 0), fx=0.25, fy=0.25)
# r = frame.shape[1] / float(rgb.shape[1])
detected_boxes = face_recognition.face_locations(
rgb, model=self.detection_model)
return rgb, detected_boxes
def __detect_with_haarcascade(self, frame):
"""Method to detecting objects with haarcascade method.
Args:
frame: Frame matrix in bgr format.
"""
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# gray = cv2.equalizeHist(gray)
reworked_boxes = []
for object_cascade in self.object_cascades:
detected_boxes = object_cascade.detectMultiScale(gray, 1.3, 5)
# Following list's member change is for face recognition format compatibility.
for box in detected_boxes:
# box[3] is x,
# box[0] is y,
# box[1] is x + w,
# box[2] is y + h.
reworked_boxes.append(
(box[1], box[0] + box[2], box[1] + box[3], box[0]))
rgb = cv2.cvtColor(
frame, cv2.COLOR_BGR2RGB) # For __recognize_things compatibility.
return rgb, reworked_boxes
def __recognize_things(self, rgb, boxes):
"""Method to recognizing objects with encodings pickle files.
Args:
rgb: Frame matrix in rgb format.
boxes: Tuple variable of locations of detected objects.
"""
encodings = face_recognition.face_encodings(rgb, boxes)
names = []
for encoding in encodings:
# attempt to match each face in the input image to our known encodings
matches = face_recognition.compare_faces(
self.recognition_data["encodings"], encoding)
name = "Unknown"
# check to see if we have found a match
if True in matches:
# find the indexes of all matched faces then initialize a
# dictionary to count the total number of times each face
# was matched
matched_idxs = [i for (i, b) in enumerate(matches) if b]
counts = {}
# loop over the matched indexes and maintain a count for
# each recognized face
for i in matched_idxs:
name = self.recognition_data["names"][i]
counts[name] = counts.get(name, 0) + 1
# determine the recognized face with the largest number
# of votes (note: in the event of an unlikely tie Python
# will select first entry in the dictionary)
name = max(counts, key=counts.get)
# update the list of names
names.append(name)
return names
def __create_tracker_by_name(self):
"""Method to creating a tracker object via type that is chosen by the user.
"""
if self.tracker_type == self.tracker_types[0]:
tracker = cv2.TrackerBoosting_create()
elif self.tracker_type == self.tracker_types[1]:
tracker = cv2.TrackerMIL_create()
elif self.tracker_type == self.tracker_types[2]:
tracker = cv2.TrackerKCF_create()
elif self.tracker_type == self.tracker_types[3]:
tracker = cv2.TrackerTLD_create()
elif self.tracker_type == self.tracker_types[4]:
tracker = cv2.TrackerMedianFlow_create()
elif self.tracker_type == self.tracker_types[5]:
tracker = cv2.TrackerGOTURN_create()
elif self.tracker_type == self.tracker_types[6]:
tracker = cv2.TrackerMOSSE_create()
elif self.tracker_type == self.tracker_types[7]:
tracker = cv2.TrackerCSRT_create()
else:
tracker = None
logger.error('Incorrect tracker name')
logger.info('Available trackers are:')
for t in self.tracker_types:
logger.info(t)
return tracker
@staticmethod
def __relocate_detected_coords(detected_boxes):
"""Method to relocating members of detected boxes from given shape to wanted shape.
Args:
detected_boxes (tuple): Top left and bottom right coordinates of the detected thing.
"""
reworked_boxes = []
for box in detected_boxes:
# box[3] is x,
# box[0] is y,
# box[1] is x + w,
# box[2] is y + h.
reworked_boxes.append(
(box[3], box[0], box[1] - box[3], box[2] - box[0]))
return reworked_boxes
def change_tracked_thing(self, file):
"""The top-level method to changing tracking objects.
Args:
file: The haarcascade trained xml file.
"""
self.object_cascades = []
ccade_xml_file = T_SYSTEM_PATH + "/haarcascade/" + file + ".xml"
self.object_cascades.append(cv2.CascadeClassifier(ccade_xml_file))
self.decider.set_db(file)
@staticmethod
def __mark_as_single_rect(frame, x, y, w, h, radius, physically_distance,
color, thickness):
"""Method to set mark_object method as drawing method with OpenCV's basic rectangle.
Args:
frame: Frame matrix.
x : the column number of the top left point of found object from the detection method.
y : the row number of the top left point of found object from the detection method.
w : the width of found object from the detection method.
h : the height of found object from the detection method.
radius: Radius of the aim.
physically_distance: Physically distance of the targeted object as pixel count.
color (tuple): Color of the drawing shape. In RGB Space.
thickness (int): Thickness of the drawing shape.
"""
cv2.rectangle(frame, (x, y), (x + w, y + h), color, thickness)
def __mark_as_partial_rect(self, frame, x, y, w, h, radius,
physically_distance, color, thickness):
"""Method to set mark_object method as drawing method with aimer's partial rect.
Args:
frame: Frame matrix.
x : the column number of the top left point of found object from the detection method.
y : the row number of the top left point of found object from the detection method.
w : the width of found object from the detection method.
h : the height of found object from the detection method.
radius: Radius of the aim.
physically_distance: Physically distance of the targeted object as pixel count.
color (tuple): Color of the drawing shape. In RGB Space.
thickness (int): Thickness of the drawing shape.
"""
self.aimer.mark_partial_rect(frame, (int(x + w / 2), int(y + h / 2)),
radius, physically_distance)
def __mark_as_rotation_arcs(self, frame, x, y, w, h, radius,
physically_distance, color, thickness):
"""Method to set mark_object method as drawing method with aimer's rotating arcs.
Args:
frame: Frame matrix.
x : the column number of the top left point of found object from the detection method.
y : the row number of the top left point of found object from the detection method.
w : the width of found object from the detection method.
h : the height of found object from the detection method.
radius: Radius of the aim.
physically_distance: Physically distance of the targeted object as pixel count.
color (tuple): Color of the drawing shape. In RGB Space.
thickness (int): Thickness of the drawing shape.
"""
self.aimer.mark_rotating_arcs(frame, (int(x + w / 2), int(y + h / 2)),
radius, physically_distance)
def __mark_as_vendor_animation(self, frame, x, y, w, h, radius,
physically_distance, color, thickness):
"""Method to set mark_object method as drawing method with aimer's rotating arcs.
Args:
frame: Frame matrix.
x : the column number of the top left point of found object from the detection method.
y : the row number of the top left point of found object from the detection method.
w : the width of found object from the detection method.
h : the height of found object from the detection method.
radius: Radius of the aim.
physically_distance: Physically distance of the targeted object as pixel count.
color (tuple): Color of the drawing shape. In RGB Space.
thickness (int): Thickness of the drawing shape.
"""
self.aimer.mark_vendor_animation(frame,
(int(x + w / 2), int(y + h / 2)),
radius, physically_distance)
def __mark_as_none(self, frame, x, y, w, h, radius, physically_distance,
color, thickness):
"""Method to set mark_object method for draw nothing.
Args:
frame: Frame matrix.
x : the column number of the top left point of found object from the detection method.
y : the row number of the top left point of found object from the detection method.
w : the width of found object from the detection method.
h : the height of found object from the detection method.
radius: Radius of the aim.
physically_distance: Physically distance of the targeted object as pixel count.
color (tuple): Color of the drawing shape. In RGB Space.
thickness (int): Thickness of the drawing shape.
"""
pass
def __get_mark_object(self, mark_found_object):
"""Method to get mark type for using when object detected.
Args:
mark_found_object (str): The mark type of the detected object.
"""
if mark_found_object == self.target_mark_types["drawings"][0]:
return self.__mark_as_single_rect
elif mark_found_object == self.target_mark_types["drawings"][1]:
return self.__mark_as_partial_rect
elif mark_found_object == self.target_mark_types["drawings"][2]:
return self.__mark_as_rotation_arcs
elif mark_found_object is None or mark_found_object is False:
return self.__mark_as_none
else:
if mark_found_object in self.target_mark_types["animations"]:
self.aimer.set_vendor_animation(mark_found_object)
return self.__mark_as_vendor_animation
def change_mark_object_to(self, mark_found_object):
"""Method to change mark type for using when object detected by given type.
Args:
mark_found_object (str): The mark type of the detected object.
"""
self.mark_object = self.__get_mark_object(mark_found_object)
@staticmethod
def __get_recognition_data(encoding_file):
"""Method to get encoded recognition data from pickle file.
Args:
encoding_file (str): The file that is keep faces's encoded data.
"""
return pickle.loads(open(encoding_file,
"rb").read()) # this is recognition_data
@dispatch(str)
def set_recognizing(self, encoding_file):
"""The top-level method to set recognition status and parameters of Vision.
Args:
encoding_file (str): The file that is keep faces's encoded data.
"""
if not encoding_file:
self.no_recognize = True
self.track = self.__track_without_recognizing
return True
self.no_recognize = False
self.recognition_data = self.__get_recognition_data(encoding_file)
self.track = self.__track_with_recognizing
@dispatch(list)
def set_recognizing(self, encoding_files):
"""The top-level method to set recognition status and parameters of Vision.
Args:
encoding_files (list): The file that is keep faces's encoded data.
"""
self.no_recognize = False
self.recognition_data = {"encodings": [], "names": []}
for file in encoding_files:
recognition_data = self.__get_recognition_data(file)
self.recognition_data["encodings"].extend(
recognition_data["encodings"])
self.recognition_data["names"].extend(recognition_data["names"])
self.track = self.__track_with_recognizing
@staticmethod
def set_object_cascades(cascade_files):
"""The top-level method to set recognition status and parameters of Vision.
Args:
cascade_files (list): Name list that keep name of haarcascade files.
"""
object_cascades = []
for cascade_file in cascade_files:
ccade_xml_file = f'{T_SYSTEM_PATH}/haarcascade/{cascade_file}.xml'
object_cascades.append(cv2.CascadeClassifier(ccade_xml_file))
return object_cascades
def get_current_frame(self):
"""Method to get current working camera frame.
"""
return self.current_frame
def __set_mqtt_receimitter(self, mqtt_receimitter):
"""Method to set mqtt_receimitter object for publishing and subscribing data echos.
Args:
mqtt_receimitter: transmit and receive data function for mqtt communication
"""
self.mqtt_receimitter = mqtt_receimitter
def start_preview(self):
"""Method to start preview of the vision without move action.
"""
self.camera.start_preview()
def stop_preview(self):
"""Method to stop preview of the vision.
"""
self.camera.stop_preview()
def terminate_active_threads(self):
"""Method to terminate active threads of vision.
"""
for thread in self.active_threads:
if thread.is_alive():
self.stop_thread = True
thread.join()
self.active_threads.clear()
self.stop_thread = False
def stop_recording(self):
"""Method to stop recording video and audio stream.
"""
if self.record:
self.recorder.stop_shoot()
def start_recording(self, task):
"""Method to start recording video and audio stream.
Args:
task: Task for the seer. Either `learn`, `track` or `secure`.
"""
if self.record:
self.recorder.start_shoot(task)
def take_shots(self):
"""Method to take shot from camera.
"""
self.recorder.take_shots()
def __release_servos(self):
"""Method to stop sending signals to servo motors pins and clean up the gpio pins.
"""
self.target_locker.stop()
self.target_locker.gpio_cleanup()
def __release_camera(self):
"""Method to stop receiving signals from the camera and stop video recording.
"""
# self.camera.release()
# self.camera.close()
pass
def __release_hearer(self):
"""Method to stop sending signals to servo motors pins and clean up the gpio pins.
"""
if self.hearer:
self.hearer.release_members()
def release_members(self):
"""Method to close all streaming and terminate vision processes.
"""
self.stop_recording()
self.terminate_active_threads()
self.__release_hearer()
self.__release_camera()
self.__release_servos()