def track_camera(name, config: CameraConfig, model_shape, detection_queue, result_connection, detected_objects_queue, process_info):
stop_event = mp.Event()
def receiveSignal(signalNumber, frame):
stop_event.set()
signal.signal(signal.SIGTERM, receiveSignal)
signal.signal(signal.SIGINT, receiveSignal)
threading.current_thread().name = f"process:{name}"
setproctitle(f"frigate.process:{name}")
listen()
frame_queue = process_info['frame_queue']
detection_enabled = process_info['detection_enabled']
frame_shape = config.frame_shape
objects_to_track = config.objects.track
object_filters = config.objects.filters
motion_detector = MotionDetector(frame_shape, config.motion)
object_detector = RemoteObjectDetector(name, '/labelmap.txt', detection_queue, result_connection, model_shape)
object_tracker = ObjectTracker(config.detect)
frame_manager = SharedMemoryFrameManager()
process_frames(name, frame_queue, frame_shape, model_shape, frame_manager, motion_detector, object_detector,
object_tracker, detected_objects_queue, process_info, objects_to_track, object_filters, detection_enabled, stop_event)
logger.info(f"{name}: exiting subprocess")
def run_detector(detection_queue, out_events: Dict[str, mp.Event], avg_speed,
start, tf_device):
print(f"Starting detection process: {os.getpid()}")
listen()
frame_manager = SharedMemoryFrameManager()
object_detector = LocalObjectDetector(tf_device=tf_device)
outputs = {}
for name in out_events.keys():
out_shm = mp.shared_memory.SharedMemory(name=f"out-{name}",
create=False)
out_np = np.ndarray((20, 6), dtype=np.float32, buffer=out_shm.buf)
outputs[name] = {'shm': out_shm, 'np': out_np}
while True:
connection_id = detection_queue.get()
input_frame = frame_manager.get(connection_id, (1, 300, 300, 3))
if input_frame is None:
continue
# detect and send the output
start.value = datetime.datetime.now().timestamp()
detections = object_detector.detect_raw(input_frame)
duration = datetime.datetime.now().timestamp() - start.value
outputs[connection_id]['np'][:] = detections[:]
out_events[connection_id].set()
start.value = 0.0
avg_speed.value = (avg_speed.value * 9 + duration) / 10
def run_detector(detection_queue, avg_speed, start):
print(f"Starting detection process: {os.getpid()}")
listen()
plasma_client = plasma.connect("/tmp/plasma")
object_detector = ObjectDetector()
while True:
object_id_str = detection_queue.get()
object_id_hash = hashlib.sha1(str.encode(object_id_str))
object_id = plasma.ObjectID(object_id_hash.digest())
object_id_out = plasma.ObjectID(
hashlib.sha1(str.encode(f"out-{object_id_str}")).digest())
input_frame = plasma_client.get(object_id, timeout_ms=0)
if input_frame is plasma.ObjectNotAvailable:
continue
# detect and put the output in the plasma store
start.value = datetime.datetime.now().timestamp()
plasma_client.put(object_detector.detect_raw(input_frame),
object_id_out)
duration = datetime.datetime.now().timestamp() - start.value
start.value = 0.0
avg_speed.value = (avg_speed.value * 9 + duration) / 10
def run_detector(
name: str,
detection_queue: mp.Queue,
out_events: Dict[str, mp.Event],
avg_speed,
start,
model_path,
model_shape,
tf_device,
num_threads,
):
threading.current_thread().name = f"detector:{name}"
logger = logging.getLogger(f"detector.{name}")
logger.info(f"Starting detection process: {os.getpid()}")
setproctitle(f"frigate.detector.{name}")
listen()
stop_event = mp.Event()
def receiveSignal(signalNumber, frame):
stop_event.set()
signal.signal(signal.SIGTERM, receiveSignal)
signal.signal(signal.SIGINT, receiveSignal)
frame_manager = SharedMemoryFrameManager()
object_detector = LocalObjectDetector(
tf_device=tf_device, model_path=model_path, num_threads=num_threads
)
outputs = {}
for name in out_events.keys():
out_shm = mp.shared_memory.SharedMemory(name=f"out-{name}", create=False)
out_np = np.ndarray((20, 6), dtype=np.float32, buffer=out_shm.buf)
outputs[name] = {"shm": out_shm, "np": out_np}
while not stop_event.is_set():
try:
connection_id = detection_queue.get(timeout=5)
except queue.Empty:
continue
input_frame = frame_manager.get(
connection_id, (1, model_shape[0], model_shape[1], 3)
)
if input_frame is None:
continue
# detect and send the output
start.value = datetime.datetime.now().timestamp()
detections = object_detector.detect_raw(input_frame)
duration = datetime.datetime.now().timestamp() - start.value
outputs[connection_id]["np"][:] = detections[:]
out_events[connection_id].set()
start.value = 0.0
avg_speed.value = (avg_speed.value * 9 + duration) / 10
def track_camera(name, config, frame_queue, frame_shape, detection_queue,
result_connection, detected_objects_queue, fps, detection_fps,
read_start, detection_frame, stop_event):
print(f"Starting process for {name}: {os.getpid()}")
listen()
detection_frame.value = 0.0
# Merge the tracked object config with the global config
camera_objects_config = config.get('objects', {})
objects_to_track = camera_objects_config.get('track', [])
object_filters = camera_objects_config.get('filters', {})
# load in the mask for object detection
if 'mask' in config:
if config['mask'].startswith('base64,'):
img = base64.b64decode(config['mask'][7:])
npimg = np.fromstring(img, dtype=np.uint8)
mask = cv2.imdecode(npimg, cv2.IMREAD_GRAYSCALE)
elif config['mask'].startswith('poly,'):
points = config['mask'].split(',')[1:]
contour = np.array([[int(points[i]),
int(points[i + 1])]
for i in range(0, len(points), 2)])
mask = np.zeros((frame_shape[0], frame_shape[1]), np.uint8)
mask[:] = 255
cv2.fillPoly(mask, pts=[contour], color=(0))
else:
mask = cv2.imread("/config/{}".format(config['mask']),
cv2.IMREAD_GRAYSCALE)
else:
mask = None
if mask is None or mask.size == 0:
mask = np.zeros((frame_shape[0], frame_shape[1]), np.uint8)
mask[:] = 255
motion_detector = MotionDetector(frame_shape, mask, resize_factor=6)
object_detector = RemoteObjectDetector(name, '/labelmap.txt',
detection_queue, result_connection)
object_tracker = ObjectTracker(10)
frame_manager = SharedMemoryFrameManager()
process_frames(name, frame_queue, frame_shape, frame_manager,
motion_detector, object_detector, object_tracker,
detected_objects_queue, fps, detection_fps, detection_frame,
objects_to_track, object_filters, mask, stop_event)
print(f"{name}: exiting subprocess")
def track_camera(name, config, global_objects_config, frame_queue, frame_shape,
detection_queue, detected_objects_queue, fps, detection_fps,
read_start, detection_frame):
print(f"Starting process for {name}: {os.getpid()}")
listen()
detection_frame.value = 0.0
# Merge the tracked object config with the global config
camera_objects_config = config.get('objects', {})
# combine tracked objects lists
objects_to_track = set().union(
global_objects_config.get('track', ['person', 'car', 'truck']),
camera_objects_config.get('track', []))
# merge object filters
global_object_filters = global_objects_config.get('filters', {})
camera_object_filters = camera_objects_config.get('filters', {})
objects_with_config = set().union(global_object_filters.keys(),
camera_object_filters.keys())
object_filters = {}
for obj in objects_with_config:
object_filters[obj] = {
**global_object_filters.get(obj, {}),
**camera_object_filters.get(obj, {})
}
frame = np.zeros(frame_shape, np.uint8)
# load in the mask for object detection
if 'mask' in config:
mask = cv2.imread("/config/{}".format(config['mask']),
cv2.IMREAD_GRAYSCALE)
else:
mask = None
if mask is None:
mask = np.zeros((frame_shape[0], frame_shape[1], 1), np.uint8)
mask[:] = 255
motion_detector = MotionDetector(frame_shape, mask, resize_factor=6)
object_detector = RemoteObjectDetector(name, '/labelmap.txt',
detection_queue)
camera_tracker_config = config.get('tracker', {
"min_hits": 1,
"max_age": 5,
"iou_threshold": 0.2
})
object_tracker = ObjectTracker(camera_tracker_config["min_hits"],
camera_tracker_config["max_age"],
camera_tracker_config["iou_threshold"])
plasma_client = PlasmaManager()
avg_wait = 0.0
fps_tracker = EventsPerSecond()
fps_tracker.start()
object_detector.fps.start()
while True:
read_start.value = datetime.datetime.now().timestamp()
frame_time = frame_queue.get()
duration = datetime.datetime.now().timestamp() - read_start.value
read_start.value = 0.0
avg_wait = (avg_wait * 99 + duration) / 100
detection_frame.value = frame_time
# Get frame from plasma store
frame = plasma_client.get(f"{name}{frame_time}")
if frame is plasma.ObjectNotAvailable:
continue
fps_tracker.update()
fps.value = fps_tracker.eps()
detection_fps.value = object_detector.fps.eps()
# look for motion
motion_boxes = motion_detector.detect(frame)
tracked_objects = object_tracker.tracked_objects.values()
# merge areas of motion that intersect with a known tracked object into a single area to look at
areas_of_interest = []
used_motion_boxes = []
for obj in tracked_objects:
x_min, y_min, x_max, y_max = obj['box']
for m_index, motion_box in enumerate(motion_boxes):
if intersection_over_union(motion_box, obj['box']) > .2:
used_motion_boxes.append(m_index)
x_min = min(obj['box'][0], motion_box[0])
y_min = min(obj['box'][1], motion_box[1])
x_max = max(obj['box'][2], motion_box[2])
y_max = max(obj['box'][3], motion_box[3])
areas_of_interest.append((x_min, y_min, x_max, y_max))
unused_motion_boxes = set(range(
0, len(motion_boxes))).difference(used_motion_boxes)
# compute motion regions
motion_regions = [
calculate_region(frame_shape, motion_boxes[i][0],
motion_boxes[i][1], motion_boxes[i][2],
motion_boxes[i][3], 1.2)
for i in unused_motion_boxes
]
# compute tracked object regions
object_regions = [
calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2)
for a in areas_of_interest
]
# merge regions with high IOU
merged_regions = motion_regions + object_regions
while True:
max_iou = 0.0
max_indices = None
region_indices = range(len(merged_regions))
for a, b in itertools.combinations(region_indices, 2):
iou = intersection_over_union(merged_regions[a],
merged_regions[b])
if iou > max_iou:
max_iou = iou
max_indices = (a, b)
if max_iou > 0.1:
a = merged_regions[max_indices[0]]
b = merged_regions[max_indices[1]]
merged_regions.append(
calculate_region(frame_shape, min(a[0], b[0]),
min(a[1], b[1]), max(a[2], b[2]),
max(a[3], b[3]), 1))
del merged_regions[max(max_indices[0], max_indices[1])]
del merged_regions[min(max_indices[0], max_indices[1])]
else:
break
# resize regions and detect
detections = []
for region in merged_regions:
tensor_input = create_tensor_input(frame, region)
region_detections = object_detector.detect(tensor_input)
for d in region_detections:
box = d[2]
size = region[2] - region[0]
x_min = int((box[1] * size) + region[0])
y_min = int((box[0] * size) + region[1])
x_max = int((box[3] * size) + region[0])
y_max = int((box[2] * size) + region[1])
det = (d[0], d[1], (x_min, y_min, x_max, y_max),
(x_max - x_min) * (y_max - y_min), region)
if filtered(det, objects_to_track, object_filters, mask):
continue
detections.append(det)
#########
# merge objects, check for clipped objects and look again up to N times
#########
refining = True
refine_count = 0
while refining and refine_count < 4:
refining = False
# group by name
detected_object_groups = defaultdict(lambda: [])
for detection in detections:
detected_object_groups[detection[0]].append(detection)
selected_objects = []
for group in detected_object_groups.values():
# apply non-maxima suppression to suppress weak, overlapping bounding boxes
boxes = [(o[2][0], o[2][1], o[2][2] - o[2][0],
o[2][3] - o[2][1]) for o in group]
confidences = [o[1] for o in group]
idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
for index in idxs:
obj = group[index[0]]
if clipped(obj, frame_shape):
box = obj[2]
# calculate a new region that will hopefully get the entire object
region = calculate_region(frame_shape, box[0], box[1],
box[2], box[3])
tensor_input = create_tensor_input(frame, region)
# run detection on new region
refined_detections = object_detector.detect(
tensor_input)
for d in refined_detections:
box = d[2]
size = region[2] - region[0]
x_min = int((box[1] * size) + region[0])
y_min = int((box[0] * size) + region[1])
x_max = int((box[3] * size) + region[0])
y_max = int((box[2] * size) + region[1])
det = (d[0], d[1], (x_min, y_min, x_max, y_max),
(x_max - x_min) * (y_max - y_min), region)
if filtered(det, objects_to_track, object_filters,
mask):
continue
selected_objects.append(det)
refining = True
else:
selected_objects.append(obj)
# set the detections list to only include top, complete objects
# and new detections
detections = selected_objects
if refining:
refine_count += 1
# now that we have refined our detections, we need to track objects
object_tracker.match_and_update(frame_time, detections)
# add to the queue
detected_objects_queue.put(
(name, frame_time, object_tracker.tracked_objects))
print(f"{name}: exiting subprocess")
