Esempio n. 1
0
def process_frames(camera_name: str,
                   frame_queue: mp.Queue,
                   frame_shape,
                   frame_manager: FrameManager,
                   motion_detector: MotionDetector,
                   object_detector: RemoteObjectDetector,
                   object_tracker: ObjectTracker,
                   detected_objects_queue: mp.Queue,
                   fps: mp.Value,
                   detection_fps: mp.Value,
                   current_frame_time: mp.Value,
                   objects_to_track: List[str],
                   object_filters: Dict,
                   mask,
                   stop_event: mp.Event,
                   exit_on_empty: bool = False):

    fps_tracker = EventsPerSecond()
    fps_tracker.start()

    while True:
        if stop_event.is_set() or (exit_on_empty and frame_queue.empty()):
            print(f"Exiting track_objects...")
            break

        try:
            frame_time = frame_queue.get(True, 10)
        except queue.Empty:
            continue

        current_frame_time.value = frame_time

        frame = frame_manager.get(f"{camera_name}{frame_time}",
                                  (frame_shape[0] * 3 // 2, frame_shape[1]))

        if frame is None:
            print(f"{camera_name}: frame {frame_time} is not in memory store.")
            continue

        fps_tracker.update()
        fps.value = fps_tracker.eps()

        # look for motion
        motion_boxes = motion_detector.detect(frame)

        tracked_object_boxes = [
            obj['box'] for obj in object_tracker.tracked_objects.values()
        ]

        # combine motion boxes with known locations of existing objects
        combined_boxes = reduce_boxes(motion_boxes + tracked_object_boxes)

        # compute regions
        regions = [
            calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2)
            for a in combined_boxes
        ]

        # combine overlapping regions
        combined_regions = reduce_boxes(regions)

        # re-compute regions
        regions = [
            calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.0)
            for a in combined_regions
        ]

        # resize regions and detect
        detections = []
        for region in regions:
            detections.extend(
                detect(object_detector, frame, region, objects_to_track,
                       object_filters, mask))

        #########
        # merge objects, check for clipped objects and look again up to 4 times
        #########
        refining = True
        refine_count = 0
        while refining and refine_count < 4:
            refining = False

            # group by name
            detected_object_groups = defaultdict(lambda: [])
            for detection in detections:
                detected_object_groups[detection[0]].append(detection)

            selected_objects = []
            for group in detected_object_groups.values():

                # apply non-maxima suppression to suppress weak, overlapping bounding boxes
                boxes = [(o[2][0], o[2][1], o[2][2] - o[2][0],
                          o[2][3] - o[2][1]) for o in group]
                confidences = [o[1] for o in group]
                idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)

                for index in idxs:
                    obj = group[index[0]]
                    if clipped(obj, frame_shape):
                        box = obj[2]
                        # calculate a new region that will hopefully get the entire object
                        region = calculate_region(frame_shape, box[0], box[1],
                                                  box[2], box[3])

                        selected_objects.extend(
                            detect(object_detector, frame, region,
                                   objects_to_track, object_filters, mask))

                        refining = True
                    else:
                        selected_objects.append(obj)
            # set the detections list to only include top, complete objects
            # and new detections
            detections = selected_objects

            if refining:
                refine_count += 1

        # now that we have refined our detections, we need to track objects
        object_tracker.match_and_update(frame_time, detections)

        # add to the queue
        detected_objects_queue.put(
            (camera_name, frame_time, object_tracker.tracked_objects))

        detection_fps.value = object_detector.fps.eps()

        frame_manager.close(f"{camera_name}{frame_time}")
Esempio n. 2
0
def track_camera(name, config, global_objects_config, frame_queue, frame_shape,
                 detection_queue, detected_objects_queue, fps, detection_fps,
                 read_start, detection_frame):
    print(f"Starting process for {name}: {os.getpid()}")
    listen()

    detection_frame.value = 0.0

    # Merge the tracked object config with the global config
    camera_objects_config = config.get('objects', {})
    # combine tracked objects lists
    objects_to_track = set().union(
        global_objects_config.get('track', ['person', 'car', 'truck']),
        camera_objects_config.get('track', []))
    # merge object filters
    global_object_filters = global_objects_config.get('filters', {})
    camera_object_filters = camera_objects_config.get('filters', {})
    objects_with_config = set().union(global_object_filters.keys(),
                                      camera_object_filters.keys())
    object_filters = {}
    for obj in objects_with_config:
        object_filters[obj] = {
            **global_object_filters.get(obj, {}),
            **camera_object_filters.get(obj, {})
        }

    frame = np.zeros(frame_shape, np.uint8)

    # load in the mask for object detection
    if 'mask' in config:
        mask = cv2.imread("/config/{}".format(config['mask']),
                          cv2.IMREAD_GRAYSCALE)
    else:
        mask = None

    if mask is None:
        mask = np.zeros((frame_shape[0], frame_shape[1], 1), np.uint8)
        mask[:] = 255

    motion_detector = MotionDetector(frame_shape, mask, resize_factor=6)
    object_detector = RemoteObjectDetector(name, '/labelmap.txt',
                                           detection_queue)

    camera_tracker_config = config.get('tracker', {
        "min_hits": 1,
        "max_age": 5,
        "iou_threshold": 0.2
    })
    object_tracker = ObjectTracker(camera_tracker_config["min_hits"],
                                   camera_tracker_config["max_age"],
                                   camera_tracker_config["iou_threshold"])

    plasma_client = PlasmaManager()
    avg_wait = 0.0
    fps_tracker = EventsPerSecond()
    fps_tracker.start()
    object_detector.fps.start()
    while True:
        read_start.value = datetime.datetime.now().timestamp()
        frame_time = frame_queue.get()
        duration = datetime.datetime.now().timestamp() - read_start.value
        read_start.value = 0.0
        avg_wait = (avg_wait * 99 + duration) / 100
        detection_frame.value = frame_time

        # Get frame from plasma store
        frame = plasma_client.get(f"{name}{frame_time}")

        if frame is plasma.ObjectNotAvailable:
            continue

        fps_tracker.update()
        fps.value = fps_tracker.eps()
        detection_fps.value = object_detector.fps.eps()

        # look for motion
        motion_boxes = motion_detector.detect(frame)

        tracked_objects = object_tracker.tracked_objects.values()

        # merge areas of motion that intersect with a known tracked object into a single area to look at
        areas_of_interest = []
        used_motion_boxes = []
        for obj in tracked_objects:
            x_min, y_min, x_max, y_max = obj['box']
            for m_index, motion_box in enumerate(motion_boxes):
                if intersection_over_union(motion_box, obj['box']) > .2:
                    used_motion_boxes.append(m_index)
                    x_min = min(obj['box'][0], motion_box[0])
                    y_min = min(obj['box'][1], motion_box[1])
                    x_max = max(obj['box'][2], motion_box[2])
                    y_max = max(obj['box'][3], motion_box[3])
            areas_of_interest.append((x_min, y_min, x_max, y_max))
        unused_motion_boxes = set(range(
            0, len(motion_boxes))).difference(used_motion_boxes)

        # compute motion regions
        motion_regions = [
            calculate_region(frame_shape, motion_boxes[i][0],
                             motion_boxes[i][1], motion_boxes[i][2],
                             motion_boxes[i][3], 1.2)
            for i in unused_motion_boxes
        ]

        # compute tracked object regions
        object_regions = [
            calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2)
            for a in areas_of_interest
        ]

        # merge regions with high IOU
        merged_regions = motion_regions + object_regions
        while True:
            max_iou = 0.0
            max_indices = None
            region_indices = range(len(merged_regions))
            for a, b in itertools.combinations(region_indices, 2):
                iou = intersection_over_union(merged_regions[a],
                                              merged_regions[b])
                if iou > max_iou:
                    max_iou = iou
                    max_indices = (a, b)
            if max_iou > 0.1:
                a = merged_regions[max_indices[0]]
                b = merged_regions[max_indices[1]]
                merged_regions.append(
                    calculate_region(frame_shape, min(a[0], b[0]),
                                     min(a[1], b[1]), max(a[2], b[2]),
                                     max(a[3], b[3]), 1))
                del merged_regions[max(max_indices[0], max_indices[1])]
                del merged_regions[min(max_indices[0], max_indices[1])]
            else:
                break

        # resize regions and detect
        detections = []
        for region in merged_regions:

            tensor_input = create_tensor_input(frame, region)

            region_detections = object_detector.detect(tensor_input)

            for d in region_detections:
                box = d[2]
                size = region[2] - region[0]
                x_min = int((box[1] * size) + region[0])
                y_min = int((box[0] * size) + region[1])
                x_max = int((box[3] * size) + region[0])
                y_max = int((box[2] * size) + region[1])
                det = (d[0], d[1], (x_min, y_min, x_max, y_max),
                       (x_max - x_min) * (y_max - y_min), region)
                if filtered(det, objects_to_track, object_filters, mask):
                    continue
                detections.append(det)

        #########
        # merge objects, check for clipped objects and look again up to N times
        #########
        refining = True
        refine_count = 0
        while refining and refine_count < 4:
            refining = False

            # group by name
            detected_object_groups = defaultdict(lambda: [])
            for detection in detections:
                detected_object_groups[detection[0]].append(detection)

            selected_objects = []
            for group in detected_object_groups.values():

                # apply non-maxima suppression to suppress weak, overlapping bounding boxes
                boxes = [(o[2][0], o[2][1], o[2][2] - o[2][0],
                          o[2][3] - o[2][1]) for o in group]
                confidences = [o[1] for o in group]
                idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)

                for index in idxs:
                    obj = group[index[0]]
                    if clipped(obj, frame_shape):
                        box = obj[2]
                        # calculate a new region that will hopefully get the entire object
                        region = calculate_region(frame_shape, box[0], box[1],
                                                  box[2], box[3])

                        tensor_input = create_tensor_input(frame, region)
                        # run detection on new region
                        refined_detections = object_detector.detect(
                            tensor_input)
                        for d in refined_detections:
                            box = d[2]
                            size = region[2] - region[0]
                            x_min = int((box[1] * size) + region[0])
                            y_min = int((box[0] * size) + region[1])
                            x_max = int((box[3] * size) + region[0])
                            y_max = int((box[2] * size) + region[1])
                            det = (d[0], d[1], (x_min, y_min, x_max, y_max),
                                   (x_max - x_min) * (y_max - y_min), region)
                            if filtered(det, objects_to_track, object_filters,
                                        mask):
                                continue
                            selected_objects.append(det)

                        refining = True
                    else:
                        selected_objects.append(obj)

            # set the detections list to only include top, complete objects
            # and new detections
            detections = selected_objects

            if refining:
                refine_count += 1

        # now that we have refined our detections, we need to track objects
        object_tracker.match_and_update(frame_time, detections)

        # add to the queue
        detected_objects_queue.put(
            (name, frame_time, object_tracker.tracked_objects))

    print(f"{name}: exiting subprocess")
Esempio n. 3
0
def track_camera(name, camera, ffmpeg_global_config, global_objects_config,
                 detection_queue, detected_objects_queue, fps, skipped_fps,
                 detection_fps):
    info(f"Starting process for {name}: {os.getpid()}")
    # info("name={} config:{} ffmpeg_global_config={} global_objects_config={}  fps={}  skipped_fps={} detection_fps={} ".format(name, config, ffmpeg_global_config, global_objects_config, fps, skipped_fps, detection_fps))
    # Merge the ffmpeg config with the global config
    config = camera.camera_conf
    ffmpeg = config.get('ffmpeg', {})
    # info(ffmpeg)
    # ffmpeg_input = get_ffmpeg_input(ffmpeg['input'])
    ffmpeg_input = camera.live_address
    # info(camera.live_address)
    # info("6666666")
    ffmpeg_global_args = ffmpeg.get('global_args',
                                    ffmpeg_global_config['global_args'])
    ffmpeg_hwaccel_args = ffmpeg.get('hwaccel_args',
                                     ffmpeg_global_config['hwaccel_args'])
    ffmpeg_input_args = ffmpeg.get('input_args',
                                   ffmpeg_global_config['input_args'])
    ffmpeg_output_args = ffmpeg.get('output_args',
                                    ffmpeg_global_config['output_args'])
    ffmpeg_cmd = (['ffmpeg'] + ffmpeg_global_args + ffmpeg_hwaccel_args +
                  ffmpeg_input_args + ['-i', ffmpeg_input] +
                  ffmpeg_output_args + ['pipe:'])

    # info(ffmpeg_cmd)

    # Merge the tracked object config with the global config
    camera_objects_config = config.get('objects', {})
    # combine tracked objects lists
    objects_to_track = set().union(
        global_objects_config.get('track', ['person', 'car', 'truck']),
        camera_objects_config.get('track', []))
    # merge object filters
    global_object_filters = global_objects_config.get('filters', {})
    camera_object_filters = camera_objects_config.get('filters', {})
    objects_with_config = set().union(global_object_filters.keys(),
                                      camera_object_filters.keys())
    object_filters = {}
    for obj in objects_with_config:
        object_filters[obj] = {
            **global_object_filters.get(obj, {}),
            **camera_object_filters.get(obj, {})
        }

    expected_fps = config['fps']
    take_frame = config.get('take_frame', 1)

    if 'width' in config and 'height' in config:
        frame_shape = (config['height'], config['width'], 3)
    else:

        frame_shape = get_frame_shape(ffmpeg_input)
        info(frame_shape)

    frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2]

    try:
        sa.delete(name)
    except:
        pass

    frame = sa.create(name, shape=frame_shape, dtype=np.uint8)

    # load in the mask for object detection
    if 'mask' in config:
        mask = cv2.imread("/config/{}".format(config['mask']),
                          cv2.IMREAD_GRAYSCALE)
    else:
        mask = None

    if mask is None:
        mask = np.zeros((frame_shape[0], frame_shape[1], 1), np.uint8)
        mask[:] = 255

    motion_detector = MotionDetector(frame_shape, mask, resize_factor=6)
    object_detector = RemoteObjectDetector(name, '/labelmap.txt',
                                           detection_queue)

    object_tracker = ObjectTracker(10)

    ffmpeg_process = start_or_restart_ffmpeg(ffmpeg_cmd, frame_size)

    plasma_client = plasma.connect("/tmp/plasma")
    frame_num = 0
    avg_wait = 0.0
    fps_tracker = EventsPerSecond()
    skipped_fps_tracker = EventsPerSecond()
    fps_tracker.start()
    skipped_fps_tracker.start()
    object_detector.fps.start()
    while True:
        start = datetime.datetime.now().timestamp()
        frame_bytes = ffmpeg_process.stdout.read(frame_size)
        duration = datetime.datetime.now().timestamp() - start
        avg_wait = (avg_wait * 99 + duration) / 100

        if not frame_bytes:
            rc = ffmpeg_process.poll()
            if rc is not None:
                info(f"{name}: ffmpeg_process exited unexpectedly with {rc}")
                ffmpeg_process = start_or_restart_ffmpeg(
                    ffmpeg_cmd, frame_size, ffmpeg_process)
                time.sleep(10)
            else:
                error(
                    f"{name}: ffmpeg_process is still running but didnt return any bytes"
                )
            continue

        # limit frame rate
        frame_num += 1
        if (frame_num % take_frame) != 0:
            continue

        fps_tracker.update()
        fps.value = fps_tracker.eps()
        detection_fps.value = object_detector.fps.eps()

        frame_time = datetime.datetime.now().timestamp()

        # Store frame in numpy array
        frame[:] = (np.frombuffer(frame_bytes, np.uint8).reshape(frame_shape))

        # look for motion
        motion_boxes = motion_detector.detect(frame)

        # skip object detection if we are below the min_fps and wait time is less than half the average
        if frame_num > 100 and fps.value < expected_fps - 1 and duration < 0.5 * avg_wait:
            skipped_fps_tracker.update()
            skipped_fps.value = skipped_fps_tracker.eps()
            continue

        skipped_fps.value = skipped_fps_tracker.eps()

        tracked_objects = object_tracker.tracked_objects.values()

        # merge areas of motion that intersect with a known tracked object into a single area to look at
        areas_of_interest = []
        used_motion_boxes = []
        for obj in tracked_objects:
            x_min, y_min, x_max, y_max = obj['box']
            for m_index, motion_box in enumerate(motion_boxes):
                if area(intersection(obj['box'],
                                     motion_box)) / area(motion_box) > .5:
                    used_motion_boxes.append(m_index)
                    x_min = min(obj['box'][0], motion_box[0])
                    y_min = min(obj['box'][1], motion_box[1])
                    x_max = max(obj['box'][2], motion_box[2])
                    y_max = max(obj['box'][3], motion_box[3])
            areas_of_interest.append((x_min, y_min, x_max, y_max))
        unused_motion_boxes = set(range(
            0, len(motion_boxes))).difference(used_motion_boxes)

        # compute motion regions
        motion_regions = [
            calculate_region(frame_shape, motion_boxes[i][0],
                             motion_boxes[i][1], motion_boxes[i][2],
                             motion_boxes[i][3], 1.2)
            for i in unused_motion_boxes
        ]

        # compute tracked object regions
        object_regions = [
            calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2)
            for a in areas_of_interest
        ]

        # merge regions with high IOU
        merged_regions = motion_regions + object_regions
        while True:
            max_iou = 0.0
            max_indices = None
            region_indices = range(len(merged_regions))
            for a, b in itertools.combinations(region_indices, 2):
                iou = intersection_over_union(merged_regions[a],
                                              merged_regions[b])
                if iou > max_iou:
                    max_iou = iou
                    max_indices = (a, b)
            if max_iou > 0.1:
                a = merged_regions[max_indices[0]]
                b = merged_regions[max_indices[1]]
                merged_regions.append(
                    calculate_region(frame_shape, min(a[0], b[0]),
                                     min(a[1], b[1]), max(a[2], b[2]),
                                     max(a[3], b[3]), 1))
                del merged_regions[max(max_indices[0], max_indices[1])]
                del merged_regions[min(max_indices[0], max_indices[1])]
            else:
                break

        # resize regions and detect
        detections = []
        for region in merged_regions:

            tensor_input = create_tensor_input(frame, region)

            region_detections = object_detector.detect(tensor_input)

            for d in region_detections:
                box = d[2]
                size = region[2] - region[0]
                x_min = int((box[1] * size) + region[0])
                y_min = int((box[0] * size) + region[1])
                x_max = int((box[3] * size) + region[0])
                y_max = int((box[2] * size) + region[1])
                det = (d[0], d[1], (x_min, y_min, x_max, y_max),
                       (x_max - x_min) * (y_max - y_min), region)
                if filtered(det, objects_to_track, object_filters, mask):
                    continue
                detections.append(det)

        #########
        # merge objects, check for clipped objects and look again up to N times
        #########
        refining = True
        refine_count = 0
        while refining and refine_count < 4:
            refining = False

            # group by name
            detected_object_groups = defaultdict(lambda: [])
            for detection in detections:
                detected_object_groups[detection[0]].append(detection)

            selected_objects = []
            for group in detected_object_groups.values():

                # apply non-maxima suppression to suppress weak, overlapping bounding boxes
                boxes = [(o[2][0], o[2][1], o[2][2] - o[2][0],
                          o[2][3] - o[2][1]) for o in group]
                confidences = [o[1] for o in group]
                idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)

                for index in idxs:
                    obj = group[index[0]]
                    if clipped(obj, frame_shape):  #obj['clipped']:
                        box = obj[2]
                        # calculate a new region that will hopefully get the entire object
                        region = calculate_region(frame_shape, box[0], box[1],
                                                  box[2], box[3])

                        tensor_input = create_tensor_input(frame, region)
                        # run detection on new region
                        refined_detections = object_detector.detect(
                            tensor_input)
                        for d in refined_detections:
                            box = d[2]
                            size = region[2] - region[0]
                            x_min = int((box[1] * size) + region[0])
                            y_min = int((box[0] * size) + region[1])
                            x_max = int((box[3] * size) + region[0])
                            y_max = int((box[2] * size) + region[1])
                            det = (d[0], d[1], (x_min, y_min, x_max, y_max),
                                   (x_max - x_min) * (y_max - y_min), region)
                            if filtered(det, objects_to_track, object_filters,
                                        mask):
                                continue
                            selected_objects.append(det)

                        refining = True
                    else:
                        selected_objects.append(obj)

            # set the detections list to only include top, complete objects
            # and new detections
            detections = selected_objects

            if refining:
                refine_count += 1

        # now that we have refined our detections, we need to track objects
        object_tracker.match_and_update(frame_time, detections)

        # put the frame in the plasma store
        object_id = hashlib.sha1(str.encode(f"{name}{frame_time}")).digest()
        plasma_client.put(frame, plasma.ObjectID(object_id))
        # add to the queue
        detected_objects_queue.put(
            (name, frame_time, object_tracker.tracked_objects))

    info(f"{name}: exiting subprocess")
Esempio n. 4
0
def process_frames(
    camera_name: str,
    frame_queue: mp.Queue,
    frame_shape,
    model_shape,
    detect_config: DetectConfig,
    frame_manager: FrameManager,
    motion_detector: MotionDetector,
    object_detector: RemoteObjectDetector,
    object_tracker: ObjectTracker,
    detected_objects_queue: mp.Queue,
    process_info: Dict,
    objects_to_track: List[str],
    object_filters,
    detection_enabled: mp.Value,
    stop_event,
    exit_on_empty: bool = False,
):

    fps = process_info["process_fps"]
    detection_fps = process_info["detection_fps"]
    current_frame_time = process_info["detection_frame"]

    fps_tracker = EventsPerSecond()
    fps_tracker.start()

    startup_scan_counter = 0

    while not stop_event.is_set():
        if exit_on_empty and frame_queue.empty():
            logger.info(f"Exiting track_objects...")
            break

        try:
            frame_time = frame_queue.get(True, 10)
        except queue.Empty:
            continue

        current_frame_time.value = frame_time

        frame = frame_manager.get(
            f"{camera_name}{frame_time}", (frame_shape[0] * 3 // 2, frame_shape[1])
        )

        if frame is None:
            logger.info(f"{camera_name}: frame {frame_time} is not in memory store.")
            continue

        # look for motion
        motion_boxes = motion_detector.detect(frame)

        regions = []

        # if detection is disabled
        if not detection_enabled.value:
            object_tracker.match_and_update(frame_time, [])
        else:
            # get stationary object ids
            # check every Nth frame for stationary objects
            # disappeared objects are not stationary
            # also check for overlapping motion boxes
            stationary_object_ids = [
                obj["id"]
                for obj in object_tracker.tracked_objects.values()
                # if there hasn't been motion for 10 frames
                if obj["motionless_count"] >= 10
                # and it isn't due for a periodic check
                and (
                    detect_config.stationary.interval == 0
                    or obj["motionless_count"] % detect_config.stationary.interval != 0
                )
                # and it hasn't disappeared
                and object_tracker.disappeared[obj["id"]] == 0
                # and it doesn't overlap with any current motion boxes
                and not intersects_any(obj["box"], motion_boxes)
            ]

            # get tracked object boxes that aren't stationary
            tracked_object_boxes = [
                obj["box"]
                for obj in object_tracker.tracked_objects.values()
                if not obj["id"] in stationary_object_ids
            ]

            # combine motion boxes with known locations of existing objects
            combined_boxes = reduce_boxes(motion_boxes + tracked_object_boxes)

            region_min_size = max(model_shape[0], model_shape[1])
            # compute regions
            regions = [
                calculate_region(
                    frame_shape,
                    a[0],
                    a[1],
                    a[2],
                    a[3],
                    region_min_size,
                    multiplier=random.uniform(1.2, 1.5),
                )
                for a in combined_boxes
            ]

            # consolidate regions with heavy overlap
            regions = [
                calculate_region(
                    frame_shape, a[0], a[1], a[2], a[3], region_min_size, multiplier=1.0
                )
                for a in reduce_boxes(regions, 0.4)
            ]

            # if starting up, get the next startup scan region
            if startup_scan_counter < 9:
                ymin = int(frame_shape[0] / 3 * startup_scan_counter / 3)
                ymax = int(frame_shape[0] / 3 + ymin)
                xmin = int(frame_shape[1] / 3 * startup_scan_counter / 3)
                xmax = int(frame_shape[1] / 3 + xmin)
                regions.append(
                    calculate_region(
                        frame_shape,
                        xmin,
                        ymin,
                        xmax,
                        ymax,
                        region_min_size,
                        multiplier=1.2,
                    )
                )
                startup_scan_counter += 1

            # resize regions and detect
            # seed with stationary objects
            detections = [
                (
                    obj["label"],
                    obj["score"],
                    obj["box"],
                    obj["area"],
                    obj["region"],
                )
                for obj in object_tracker.tracked_objects.values()
                if obj["id"] in stationary_object_ids
            ]

            for region in regions:
                detections.extend(
                    detect(
                        object_detector,
                        frame,
                        model_shape,
                        region,
                        objects_to_track,
                        object_filters,
                    )
                )

            #########
            # merge objects, check for clipped objects and look again up to 4 times
            #########
            refining = len(regions) > 0
            refine_count = 0
            while refining and refine_count < 4:
                refining = False

                # group by name
                detected_object_groups = defaultdict(lambda: [])
                for detection in detections:
                    detected_object_groups[detection[0]].append(detection)

                selected_objects = []
                for group in detected_object_groups.values():

                    # apply non-maxima suppression to suppress weak, overlapping bounding boxes
                    boxes = [
                        (o[2][0], o[2][1], o[2][2] - o[2][0], o[2][3] - o[2][1])
                        for o in group
                    ]
                    confidences = [o[1] for o in group]
                    idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)

                    for index in idxs:
                        obj = group[index[0]]
                        if clipped(obj, frame_shape):
                            box = obj[2]
                            # calculate a new region that will hopefully get the entire object
                            region = calculate_region(
                                frame_shape,
                                box[0],
                                box[1],
                                box[2],
                                box[3],
                                region_min_size,
                            )

                            regions.append(region)

                            selected_objects.extend(
                                detect(
                                    object_detector,
                                    frame,
                                    model_shape,
                                    region,
                                    objects_to_track,
                                    object_filters,
                                )
                            )

                            refining = True
                        else:
                            selected_objects.append(obj)
                # set the detections list to only include top, complete objects
                # and new detections
                detections = selected_objects

                if refining:
                    refine_count += 1

            ## drop detections that overlap too much
            consolidated_detections = []

            # if detection was run on this frame, consolidate
            if len(regions) > 0:
                # group by name
                detected_object_groups = defaultdict(lambda: [])
                for detection in detections:
                    detected_object_groups[detection[0]].append(detection)

                # loop over detections grouped by label
                for group in detected_object_groups.values():
                    # if the group only has 1 item, skip
                    if len(group) == 1:
                        consolidated_detections.append(group[0])
                        continue

                    # sort smallest to largest by area
                    sorted_by_area = sorted(group, key=lambda g: g[3])

                    for current_detection_idx in range(0, len(sorted_by_area)):
                        current_detection = sorted_by_area[current_detection_idx][2]
                        overlap = 0
                        for to_check_idx in range(
                            min(current_detection_idx + 1, len(sorted_by_area)),
                            len(sorted_by_area),
                        ):
                            to_check = sorted_by_area[to_check_idx][2]
                            # if 90% of smaller detection is inside of another detection, consolidate
                            if (
                                area(intersection(current_detection, to_check))
                                / area(current_detection)
                                > 0.9
                            ):
                                overlap = 1
                                break
                        if overlap == 0:
                            consolidated_detections.append(
                                sorted_by_area[current_detection_idx]
                            )
                # now that we have refined our detections, we need to track objects
                object_tracker.match_and_update(frame_time, consolidated_detections)
            # else, just update the frame times for the stationary objects
            else:
                object_tracker.update_frame_times(frame_time)

        # add to the queue if not full
        if detected_objects_queue.full():
            frame_manager.delete(f"{camera_name}{frame_time}")
            continue
        else:
            fps_tracker.update()
            fps.value = fps_tracker.eps()
            detected_objects_queue.put(
                (
                    camera_name,
                    frame_time,
                    object_tracker.tracked_objects,
                    motion_boxes,
                    regions,
                )
            )
            detection_fps.value = object_detector.fps.eps()
            frame_manager.close(f"{camera_name}{frame_time}")