def reduce_boxes(boxes, iou_threshold=0.0): clusters = [] for box in boxes: matched = 0 for cluster in clusters: if intersection_over_union(box, cluster) > iou_threshold: matched = 1 cluster[0] = min(cluster[0], box[0]) cluster[1] = min(cluster[1], box[1]) cluster[2] = max(cluster[2], box[2]) cluster[3] = max(cluster[3], box[3]) if not matched: clusters.append(list(box)) return [tuple(c) for c in clusters]
def update_position(self, id, box): position = self.positions[id] position_box = ( position["xmin"], position["ymin"], position["xmax"], position["ymax"], ) xmin, ymin, xmax, ymax = box iou = intersection_over_union(position_box, box) # if the iou drops below the threshold # assume the object has moved to a new position and reset the computed box if iou < 0.6: self.positions[id] = { "xmins": [xmin], "ymins": [ymin], "xmaxs": [xmax], "ymaxs": [ymax], "xmin": xmin, "ymin": ymin, "xmax": xmax, "ymax": ymax, } return False # if there are less than 10 entries for the position, add the bounding box # and recompute the position box if len(position["xmins"]) < 10: position["xmins"].append(xmin) position["ymins"].append(ymin) position["xmaxs"].append(xmax) position["ymaxs"].append(ymax) # by using percentiles here, we hopefully remove outliers position["xmin"] = np.percentile(position["xmins"], 15) position["ymin"] = np.percentile(position["ymins"], 15) position["xmax"] = np.percentile(position["xmaxs"], 85) position["ymax"] = np.percentile(position["ymaxs"], 85) return True
def track_camera(name, camera, ffmpeg_global_config, global_objects_config, detection_queue, detected_objects_queue, fps, skipped_fps, detection_fps): info(f"Starting process for {name}: {os.getpid()}") # info("name={} config:{} ffmpeg_global_config={} global_objects_config={} fps={} skipped_fps={} detection_fps={} ".format(name, config, ffmpeg_global_config, global_objects_config, fps, skipped_fps, detection_fps)) # Merge the ffmpeg config with the global config config = camera.camera_conf ffmpeg = config.get('ffmpeg', {}) # info(ffmpeg) # ffmpeg_input = get_ffmpeg_input(ffmpeg['input']) ffmpeg_input = camera.live_address # info(camera.live_address) # info("6666666") ffmpeg_global_args = ffmpeg.get('global_args', ffmpeg_global_config['global_args']) ffmpeg_hwaccel_args = ffmpeg.get('hwaccel_args', ffmpeg_global_config['hwaccel_args']) ffmpeg_input_args = ffmpeg.get('input_args', ffmpeg_global_config['input_args']) ffmpeg_output_args = ffmpeg.get('output_args', ffmpeg_global_config['output_args']) ffmpeg_cmd = (['ffmpeg'] + ffmpeg_global_args + ffmpeg_hwaccel_args + ffmpeg_input_args + ['-i', ffmpeg_input] + ffmpeg_output_args + ['pipe:']) # info(ffmpeg_cmd) # Merge the tracked object config with the global config camera_objects_config = config.get('objects', {}) # combine tracked objects lists objects_to_track = set().union( global_objects_config.get('track', ['person', 'car', 'truck']), camera_objects_config.get('track', [])) # merge object filters global_object_filters = global_objects_config.get('filters', {}) camera_object_filters = camera_objects_config.get('filters', {}) objects_with_config = set().union(global_object_filters.keys(), camera_object_filters.keys()) object_filters = {} for obj in objects_with_config: object_filters[obj] = { **global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {}) } expected_fps = config['fps'] take_frame = config.get('take_frame', 1) if 'width' in config and 'height' in config: frame_shape = (config['height'], config['width'], 3) else: frame_shape = get_frame_shape(ffmpeg_input) info(frame_shape) frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2] try: sa.delete(name) except: pass frame = sa.create(name, shape=frame_shape, dtype=np.uint8) # load in the mask for object detection if 'mask' in config: mask = cv2.imread("/config/{}".format(config['mask']), cv2.IMREAD_GRAYSCALE) else: mask = None if mask is None: mask = np.zeros((frame_shape[0], frame_shape[1], 1), np.uint8) mask[:] = 255 motion_detector = MotionDetector(frame_shape, mask, resize_factor=6) object_detector = RemoteObjectDetector(name, '/labelmap.txt', detection_queue) object_tracker = ObjectTracker(10) ffmpeg_process = start_or_restart_ffmpeg(ffmpeg_cmd, frame_size) plasma_client = plasma.connect("/tmp/plasma") frame_num = 0 avg_wait = 0.0 fps_tracker = EventsPerSecond() skipped_fps_tracker = EventsPerSecond() fps_tracker.start() skipped_fps_tracker.start() object_detector.fps.start() while True: start = datetime.datetime.now().timestamp() frame_bytes = ffmpeg_process.stdout.read(frame_size) duration = datetime.datetime.now().timestamp() - start avg_wait = (avg_wait * 99 + duration) / 100 if not frame_bytes: rc = ffmpeg_process.poll() if rc is not None: info(f"{name}: ffmpeg_process exited unexpectedly with {rc}") ffmpeg_process = start_or_restart_ffmpeg( ffmpeg_cmd, frame_size, ffmpeg_process) time.sleep(10) else: error( f"{name}: ffmpeg_process is still running but didnt return any bytes" ) continue # limit frame rate frame_num += 1 if (frame_num % take_frame) != 0: continue fps_tracker.update() fps.value = fps_tracker.eps() detection_fps.value = object_detector.fps.eps() frame_time = datetime.datetime.now().timestamp() # Store frame in numpy array frame[:] = (np.frombuffer(frame_bytes, np.uint8).reshape(frame_shape)) # look for motion motion_boxes = motion_detector.detect(frame) # skip object detection if we are below the min_fps and wait time is less than half the average if frame_num > 100 and fps.value < expected_fps - 1 and duration < 0.5 * avg_wait: skipped_fps_tracker.update() skipped_fps.value = skipped_fps_tracker.eps() continue skipped_fps.value = skipped_fps_tracker.eps() tracked_objects = object_tracker.tracked_objects.values() # merge areas of motion that intersect with a known tracked object into a single area to look at areas_of_interest = [] used_motion_boxes = [] for obj in tracked_objects: x_min, y_min, x_max, y_max = obj['box'] for m_index, motion_box in enumerate(motion_boxes): if area(intersection(obj['box'], motion_box)) / area(motion_box) > .5: used_motion_boxes.append(m_index) x_min = min(obj['box'][0], motion_box[0]) y_min = min(obj['box'][1], motion_box[1]) x_max = max(obj['box'][2], motion_box[2]) y_max = max(obj['box'][3], motion_box[3]) areas_of_interest.append((x_min, y_min, x_max, y_max)) unused_motion_boxes = set(range( 0, len(motion_boxes))).difference(used_motion_boxes) # compute motion regions motion_regions = [ calculate_region(frame_shape, motion_boxes[i][0], motion_boxes[i][1], motion_boxes[i][2], motion_boxes[i][3], 1.2) for i in unused_motion_boxes ] # compute tracked object regions object_regions = [ calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2) for a in areas_of_interest ] # merge regions with high IOU merged_regions = motion_regions + object_regions while True: max_iou = 0.0 max_indices = None region_indices = range(len(merged_regions)) for a, b in itertools.combinations(region_indices, 2): iou = intersection_over_union(merged_regions[a], merged_regions[b]) if iou > max_iou: max_iou = iou max_indices = (a, b) if max_iou > 0.1: a = merged_regions[max_indices[0]] b = merged_regions[max_indices[1]] merged_regions.append( calculate_region(frame_shape, min(a[0], b[0]), min(a[1], b[1]), max(a[2], b[2]), max(a[3], b[3]), 1)) del merged_regions[max(max_indices[0], max_indices[1])] del merged_regions[min(max_indices[0], max_indices[1])] else: break # resize regions and detect detections = [] for region in merged_regions: tensor_input = create_tensor_input(frame, region) region_detections = object_detector.detect(tensor_input) for d in region_detections: box = d[2] size = region[2] - region[0] x_min = int((box[1] * size) + region[0]) y_min = int((box[0] * size) + region[1]) x_max = int((box[3] * size) + region[0]) y_max = int((box[2] * size) + region[1]) det = (d[0], d[1], (x_min, y_min, x_max, y_max), (x_max - x_min) * (y_max - y_min), region) if filtered(det, objects_to_track, object_filters, mask): continue detections.append(det) ######### # merge objects, check for clipped objects and look again up to N times ######### refining = True refine_count = 0 while refining and refine_count < 4: refining = False # group by name detected_object_groups = defaultdict(lambda: []) for detection in detections: detected_object_groups[detection[0]].append(detection) selected_objects = [] for group in detected_object_groups.values(): # apply non-maxima suppression to suppress weak, overlapping bounding boxes boxes = [(o[2][0], o[2][1], o[2][2] - o[2][0], o[2][3] - o[2][1]) for o in group] confidences = [o[1] for o in group] idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4) for index in idxs: obj = group[index[0]] if clipped(obj, frame_shape): #obj['clipped']: box = obj[2] # calculate a new region that will hopefully get the entire object region = calculate_region(frame_shape, box[0], box[1], box[2], box[3]) tensor_input = create_tensor_input(frame, region) # run detection on new region refined_detections = object_detector.detect( tensor_input) for d in refined_detections: box = d[2] size = region[2] - region[0] x_min = int((box[1] * size) + region[0]) y_min = int((box[0] * size) + region[1]) x_max = int((box[3] * size) + region[0]) y_max = int((box[2] * size) + region[1]) det = (d[0], d[1], (x_min, y_min, x_max, y_max), (x_max - x_min) * (y_max - y_min), region) if filtered(det, objects_to_track, object_filters, mask): continue selected_objects.append(det) refining = True else: selected_objects.append(obj) # set the detections list to only include top, complete objects # and new detections detections = selected_objects if refining: refine_count += 1 # now that we have refined our detections, we need to track objects object_tracker.match_and_update(frame_time, detections) # put the frame in the plasma store object_id = hashlib.sha1(str.encode(f"{name}{frame_time}")).digest() plasma_client.put(frame, plasma.ObjectID(object_id)) # add to the queue detected_objects_queue.put( (name, frame_time, object_tracker.tracked_objects)) info(f"{name}: exiting subprocess")
def track_camera(name, config, global_objects_config, frame_queue, frame_shape, detection_queue, detected_objects_queue, fps, detection_fps, read_start, detection_frame): print(f"Starting process for {name}: {os.getpid()}") listen() detection_frame.value = 0.0 # Merge the tracked object config with the global config camera_objects_config = config.get('objects', {}) # combine tracked objects lists objects_to_track = set().union( global_objects_config.get('track', ['person', 'car', 'truck']), camera_objects_config.get('track', [])) # merge object filters global_object_filters = global_objects_config.get('filters', {}) camera_object_filters = camera_objects_config.get('filters', {}) objects_with_config = set().union(global_object_filters.keys(), camera_object_filters.keys()) object_filters = {} for obj in objects_with_config: object_filters[obj] = { **global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {}) } frame = np.zeros(frame_shape, np.uint8) # load in the mask for object detection if 'mask' in config: mask = cv2.imread("/config/{}".format(config['mask']), cv2.IMREAD_GRAYSCALE) else: mask = None if mask is None: mask = np.zeros((frame_shape[0], frame_shape[1], 1), np.uint8) mask[:] = 255 motion_detector = MotionDetector(frame_shape, mask, resize_factor=6) object_detector = RemoteObjectDetector(name, '/labelmap.txt', detection_queue) camera_tracker_config = config.get('tracker', { "min_hits": 1, "max_age": 5, "iou_threshold": 0.2 }) object_tracker = ObjectTracker(camera_tracker_config["min_hits"], camera_tracker_config["max_age"], camera_tracker_config["iou_threshold"]) plasma_client = PlasmaManager() avg_wait = 0.0 fps_tracker = EventsPerSecond() fps_tracker.start() object_detector.fps.start() while True: read_start.value = datetime.datetime.now().timestamp() frame_time = frame_queue.get() duration = datetime.datetime.now().timestamp() - read_start.value read_start.value = 0.0 avg_wait = (avg_wait * 99 + duration) / 100 detection_frame.value = frame_time # Get frame from plasma store frame = plasma_client.get(f"{name}{frame_time}") if frame is plasma.ObjectNotAvailable: continue fps_tracker.update() fps.value = fps_tracker.eps() detection_fps.value = object_detector.fps.eps() # look for motion motion_boxes = motion_detector.detect(frame) tracked_objects = object_tracker.tracked_objects.values() # merge areas of motion that intersect with a known tracked object into a single area to look at areas_of_interest = [] used_motion_boxes = [] for obj in tracked_objects: x_min, y_min, x_max, y_max = obj['box'] for m_index, motion_box in enumerate(motion_boxes): if intersection_over_union(motion_box, obj['box']) > .2: used_motion_boxes.append(m_index) x_min = min(obj['box'][0], motion_box[0]) y_min = min(obj['box'][1], motion_box[1]) x_max = max(obj['box'][2], motion_box[2]) y_max = max(obj['box'][3], motion_box[3]) areas_of_interest.append((x_min, y_min, x_max, y_max)) unused_motion_boxes = set(range( 0, len(motion_boxes))).difference(used_motion_boxes) # compute motion regions motion_regions = [ calculate_region(frame_shape, motion_boxes[i][0], motion_boxes[i][1], motion_boxes[i][2], motion_boxes[i][3], 1.2) for i in unused_motion_boxes ] # compute tracked object regions object_regions = [ calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2) for a in areas_of_interest ] # merge regions with high IOU merged_regions = motion_regions + object_regions while True: max_iou = 0.0 max_indices = None region_indices = range(len(merged_regions)) for a, b in itertools.combinations(region_indices, 2): iou = intersection_over_union(merged_regions[a], merged_regions[b]) if iou > max_iou: max_iou = iou max_indices = (a, b) if max_iou > 0.1: a = merged_regions[max_indices[0]] b = merged_regions[max_indices[1]] merged_regions.append( calculate_region(frame_shape, min(a[0], b[0]), min(a[1], b[1]), max(a[2], b[2]), max(a[3], b[3]), 1)) del merged_regions[max(max_indices[0], max_indices[1])] del merged_regions[min(max_indices[0], max_indices[1])] else: break # resize regions and detect detections = [] for region in merged_regions: tensor_input = create_tensor_input(frame, region) region_detections = object_detector.detect(tensor_input) for d in region_detections: box = d[2] size = region[2] - region[0] x_min = int((box[1] * size) + region[0]) y_min = int((box[0] * size) + region[1]) x_max = int((box[3] * size) + region[0]) y_max = int((box[2] * size) + region[1]) det = (d[0], d[1], (x_min, y_min, x_max, y_max), (x_max - x_min) * (y_max - y_min), region) if filtered(det, objects_to_track, object_filters, mask): continue detections.append(det) ######### # merge objects, check for clipped objects and look again up to N times ######### refining = True refine_count = 0 while refining and refine_count < 4: refining = False # group by name detected_object_groups = defaultdict(lambda: []) for detection in detections: detected_object_groups[detection[0]].append(detection) selected_objects = [] for group in detected_object_groups.values(): # apply non-maxima suppression to suppress weak, overlapping bounding boxes boxes = [(o[2][0], o[2][1], o[2][2] - o[2][0], o[2][3] - o[2][1]) for o in group] confidences = [o[1] for o in group] idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4) for index in idxs: obj = group[index[0]] if clipped(obj, frame_shape): box = obj[2] # calculate a new region that will hopefully get the entire object region = calculate_region(frame_shape, box[0], box[1], box[2], box[3]) tensor_input = create_tensor_input(frame, region) # run detection on new region refined_detections = object_detector.detect( tensor_input) for d in refined_detections: box = d[2] size = region[2] - region[0] x_min = int((box[1] * size) + region[0]) y_min = int((box[0] * size) + region[1]) x_max = int((box[3] * size) + region[0]) y_max = int((box[2] * size) + region[1]) det = (d[0], d[1], (x_min, y_min, x_max, y_max), (x_max - x_min) * (y_max - y_min), region) if filtered(det, objects_to_track, object_filters, mask): continue selected_objects.append(det) refining = True else: selected_objects.append(obj) # set the detections list to only include top, complete objects # and new detections detections = selected_objects if refining: refine_count += 1 # now that we have refined our detections, we need to track objects object_tracker.match_and_update(frame_time, detections) # add to the queue detected_objects_queue.put( (name, frame_time, object_tracker.tracked_objects)) print(f"{name}: exiting subprocess")