def test_group_by(self): is1 = IntervalSet([ Interval(Bounds3D(0, 1)), Interval(Bounds3D(1, 2)), Interval(Bounds3D(0, 2)), Interval(Bounds3D(1, 3)), ]) def merge_intervals(k, intervals): merged = intervals.fold( lambda i1, i2: Interval(i1['bounds'].span(i2['bounds']))) merged['payload'] = intervals return merged is2 = is1.group_by(lambda i: int(i['t2']) % 2, merge_intervals) target = IntervalSet([ Interval(Bounds3D(0, 3), payload=IntervalSet([ Interval(Bounds3D(0, 1)), Interval(Bounds3D(1, 3)), ])), Interval(Bounds3D(0, 2), payload=IntervalSet([ Interval(Bounds3D(0, 2)), Interval(Bounds3D(1, 2)), ])), ]) self.assertIntervalSetEq(is2, target, self.compare_interval_sets_in_payload())
def test_len(self): is1 = IntervalSet([ Interval(Bounds3D(1, 5)), Interval(Bounds3D(10, 22)), ]) self.assertEqual(len(is1), 2) self.assertEqual(is1.size(), 2)
def test_collect_by_interval(self): c = TestIntervalSetMapping.get_collection() d = IntervalSetMapping({1: IntervalSet([ Interval(Bounds3D(t,t)) for t in range(1, 100)])}) e = c.collect_by_interval(d, Bounds3D.T(overlaps()), filter_empty=False, window=0) self.assertEqual(e.keys(), c.keys())
def test_match_multiple_solutions(self): left_box_frame_1 = Interval(Bounds3D(1, 1, 0.1, 0.4, 0.4, 0.8)) right_box_frame_1 = Interval(Bounds3D(1, 1, 0.6, 0.9, 0.3, 0.7)) bottom_left_box_frame_2 = Interval(Bounds3D(2, 2, 0.1, 0.3, 0.8, 0.9)) top_right_box_frame_2 = Interval(Bounds3D(2, 2, 0.5, 0.7, 0.2, 0.7)) is1 = IntervalSet([ left_box_frame_1, right_box_frame_1, bottom_left_box_frame_2, top_right_box_frame_2, Interval(Bounds3D(3, 3)), ]) pattern = [ ( ["left", "right"], [ # Two boxes on left and right on same frame Bounds3D.T(equal()), Bounds3D.XY(left_of()), ]), ] results = is1.match(pattern, exact=False) self.assertEqual(len(results), 2) # Add single interval constraints. pattern = pattern + [ (["left"], [Bounds3D.XY(height_at_least(0.3))]), (["right"], [Bounds3D.XY(height_at_least(0.3))]), ] results = is1.match(pattern, exact=False) self.assertEqual(len(results), 1) result = results[0] self.assertIntervalsEq(left_box_frame_1, result['left']) self.assertIntervalsEq(right_box_frame_1, result['right'])
def test_filter(self): is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2), Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3), ]) is1 = is1.filter(lambda i: i['payload'] > 2) target = IntervalSet([ Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3), ]) self.assertIntervalSetEq(is1, target, eq)
def test_fold(self): def fold_fn(acc, i): return acc + (i['bounds'].length() * i['bounds'].width() * i['bounds'].height()) is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2), Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3), ]) self.assertAlmostEqual(is1.fold(fold_fn, 0), 1.375)
def test_join_with_optimization_window(self): is1 = IntervalSet( [Interval(Bounds3D(t, t + 1), t) for t in range(100)]) is2 = IntervalSet([Interval(Bounds3D(t, t), t) for t in range(100)]) is3 = is1.join(is2, Bounds3D.T(before(max_dist=1)), lambda i1, i2: Interval(i1['bounds'].span(i2['bounds']), i2['payload']), window=1) target = IntervalSet( [Interval(Bounds3D(t, t + 2), t + 2) for t in range(98)] + [Interval(Bounds3D(t, t + 1), t + 1) for t in range(99)]) self.assertIntervalSetEq(is3, target, eq)
def test_fold_custom_sortkey(self): def sortkey(i): return (i['x1'], i['x2']) def fold_fn(acc, i): acc.append(i['payload']) return acc is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 0.1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2), Interval(Bounds3D(0.1, 0.4, 0, 1, 0, 0.5), 3), ]) self.assertListEqual(is1.fold(fold_fn, [], sortkey), [1, 3, 2])
def list_to_IntervalSetMapping(interval_list): ism = {} for video_id, start, end, duration in interval_list: if not video_id in ism: ism[video_id] = [] ism[video_id].append(Interval(Bounds3D(start, end))) return IntervalSetMapping({video_id: IntervalSet(intervalSet) for video_id, intervalSet in ism.items()})
def flatten_fn(intrvl): rv = [] try: detections = intrvl.payload[self.result_key] except KeyError: raise KeyError( f"Cannot find {self.result_key} in input payload. Did you run object detection on the input stream?" ) for box, score, class_name in zip(detections['detection_boxes'], detections['detection_scores'], detections['detection_names']): if score < self.confidence: break if (not self.black_list and class_name in self.targets) or ( self.black_list and class_name not in self.targets): # create new patch top, left, bottom, right = box # TF return between 0~1 # the following arithmetic should be correct even if `intrvl` is not full frame. new_bounds = Bounds3D( intrvl['t1'], intrvl['t2'], intrvl['x1'] + intrvl.bounds.width() * left, intrvl['x1'] + intrvl.bounds.width() * right, intrvl['y1'] + intrvl.bounds.height() * top, intrvl['y1'] + intrvl.bounds.height() * bottom) new_patch = ImageInterval(new_bounds, root=intrvl.root) rv.append(new_patch) rv.sort(key=lambda i: i.bounds) return rv
def dilate_car(icar: Interval) -> Interval: carh, carw = _height(icar), _width(icar) new_bounds = Bounds3D(t1=int(max(0, icar['t1'] - fps)), t2=int(min(frame_count, icar['t2'] + fps)), x1=max(0, icar['x1'] - carw), x2=min(1, icar['x2'] + carw), y1=max(0, icar['y1'] - carh), y2=min(1, icar['y2'] + carh)) return Interval(new_bounds)
def split_fn(i): output = [] t = i.copy() while t['bounds'].length() > 5: output.append(Interval(Bounds3D(t['t1'], t['t1'] + 5))) t['t1'] = t['t1'] + 5 if t['bounds'].length() > 0: output.append(t) return IntervalSet(output)
def average_space_span_time(list_of_box): ret_bounds = Bounds3D( t1=min([b['t1'] for b in list_of_box]), t2=max([b['t2'] for b in list_of_box]), ) for key in ('x1', 'x2', 'y1', 'y2'): ret_bounds[key] = np.mean([b[key] for b in list_of_box]) return ret_bounds
def test_map(self): def expand_to_frame(intrvl): new_bounds = intrvl['bounds'].copy() new_bounds['x1'] = 0 new_bounds['x2'] = 1 new_bounds['y1'] = 0 new_bounds['y2'] = 1 return Interval(new_bounds, intrvl['payload']) is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0.3, 0.4, 0.5, 0.6)), Interval(Bounds3D(0, 0.5, 0.2, 0.3, 0.5, 0.6)) ]) is1 = is1.map(expand_to_frame) target = IntervalSet([ Interval(Bounds3D(0, 0.5, 0, 1, 0, 1)), Interval(Bounds3D(0, 1, 0, 1, 0, 1)), ]) self.assertIntervalSetEq(is1, target)
def bounds_parser(item): args = [ key_accessor(item, schema_final['t1']), key_accessor(item, schema_final['t2']) ] kwargs = {} for k in ['x1', 'x2', 'y1', 'y2']: if k in schema_final: kwargs[k] = key_accessor(item, schema_final[k]) return Bounds3D(*args, **kwargs)
def dilate_op(i:ImageInterval) -> ImageInterval: w, h = _width(i), _height(i) new_bounds = Bounds3D( i['t1'], i['t2'], x1=max(0, i['x1']-w*3), x2=min(1, i['x2']+w*3), y1=max(0, i['y1']-h/2), y2=min(1, i['y2']+h/2) ) return ImageInterval(new_bounds, None, root=i.root)
def test_minus_everything(self): is1 = IntervalSet( [Interval(Bounds3D(1, 10)), Interval(Bounds3D(3, 15))]) is2 = IntervalSet([ Interval(Bounds3D(2, 2.5)), Interval(Bounds3D(2, 2.7)), Interval(Bounds3D(2.9, 3.5)), Interval(Bounds3D(3.5, 3.6)), Interval(Bounds3D(5, 7)), Interval(Bounds3D(9, 12)), ]) is3 = is2.minus(is1) self.assertIntervalSetEq(is3, IntervalSet([]), eq)
def test_minus_with_single_frame(self): is1 = IntervalSet([ Interval(Bounds3D(1, 1)), Interval(Bounds3D(3, 3)), Interval(Bounds3D(4, 4)), Interval(Bounds3D(7, 7)), Interval(Bounds3D(10, 10)), ]) is2 = IntervalSet([ Interval(Bounds3D(1, 3)), Interval(Bounds3D(5, 8)), Interval(Bounds3D(9, 9)), ]) is3 = is1.minus(is2) target = IntervalSet([ Interval(Bounds3D(4, 4)), Interval(Bounds3D(10, 10)), ]) self.assertIntervalSetEq(is3, target) is4 = is2.minus(is1) self.assertIntervalSetEq(is4, is2)
def test_minus_against_nothing(self): is1 = IntervalSet([ Interval(Bounds3D(1, 10, 0, 0.5, 0.2, 0.8), 1), Interval(Bounds3D(3, 15, 0, 1, 0, 1), 2) ]) is2 = IntervalSet([ Interval(Bounds3D(20, 20.5)), Interval(Bounds3D(20, 20.7)), Interval(Bounds3D(20.9, 23.5)), Interval(Bounds3D(23.5, 23.6)), Interval(Bounds3D(25, 27)), Interval(Bounds3D(29, 32)), ]) is3 = is1.minus(is2) self.assertIntervalSetEq(is3, is1, eq)
def test_filter_against(self): is1 = IntervalSet([ Interval(Bounds3D(0, 1)), Interval(Bounds3D(2, 5)), Interval(Bounds3D(6, 7)), ]) is2 = IntervalSet([ Interval(Bounds3D(1, 1)), Interval(Bounds3D(7, 7)), ]) # Take only intervals in is1 that overlaps with some interval in is2 is3 = is1.filter_against(is2, Bounds3D.T(overlaps()), window=0) self.assertIntervalSetEq( is3, IntervalSet([Interval(Bounds3D(0, 1)), Interval(Bounds3D(6, 7))]))
def test_group_by_axis(self): default_bounds = Bounds3D(0, 1, 0, 1, 0, 1) intervals_1 = [ Interval(Bounds3D(1, 1, 0.4, 0.5, 0.6, 0.8), 1), Interval(Bounds3D(1, 1, 0.1, 0.2, 0.2, 0.3), 2), Interval(Bounds3D(1, 1, 0.3, 0.5, 0.1, 0.5), 3), ] intervals_2 = [ Interval(Bounds3D(2, 2, 0.3, 0.5, 0.6, 0.8), 11), Interval(Bounds3D(2, 2, 0.2, 0.3, 0.2, 0.9), 12), Interval(Bounds3D(2, 2, 0.3, 0.7, 0, 0.5), 13), ] is1 = IntervalSet(intervals_1 + intervals_2) target = IntervalSet([ Interval(Bounds3D(1, 1), payload=IntervalSet(intervals_1)), Interval(Bounds3D(2, 2), payload=IntervalSet(intervals_2)), ]) self.assertIntervalSetEq( is1.group_by_axis(('t1', 't2'), default_bounds), target, self.compare_interval_sets_in_payload())
def merge_candidate(iperson: Interval, icar: Interval) -> Interval: carh, carw = _height(icar), _width(icar) new_bounds = Bounds3D(t1=int(max(0, icar['t1'] - fps * 3)), t2=int(max(0, iperson['t1'] + fps * 3)), x1=max(0, icar['x1'] - carw), x2=min(1, icar['x2'] + carw), y1=max(0, icar['y1'] - carh), y2=min(1, icar['y2'] + carh)) new_payload = { 'traj_person': iperson.payload['traj_person'], 'stopped_car': icar } return Interval(new_bounds, new_payload)
def test_union(self): is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 1), ]) is2 = IntervalSet([ Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 2), Interval(Bounds3D(0, 1, 0, 1, 0, 1), 2), ]) is3 = is1.union(is2) target = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 1, 0, 1, 0, 1), 2), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 1), Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 2), ]) self.assertIntervalSetEq(is3, target, eq)
def test_match(self): left_box_frame_1 = Interval(Bounds3D(1, 1, 0.1, 0.4, 0.4, 0.8)) right_box_frame_1 = Interval(Bounds3D(1, 1, 0.6, 0.9, 0.3, 0.7)) bottom_left_box_frame_2 = Interval(Bounds3D(2, 2, 0.1, 0.3, 0.8, 0.9)) top_right_box_frame_2 = Interval(Bounds3D(2, 2, 0.5, 0.7, 0.2, 0.7)) is1 = IntervalSet([ left_box_frame_1, right_box_frame_1, bottom_left_box_frame_2, top_right_box_frame_2 ]) pattern = [ ( ["left", "right"], [ # Two boxes on left and right on same frame Bounds3D.T(equal()), Bounds3D.XY(left_of()), ]), ( ["top", "bottom"], [ # Two boxes on top and bottom on overlapping frame Bounds3D.T(equal()), Bounds3D.XY(above()), ]), ( ["left", "top"], [ # Left-Right pattern comes before Top-Bottom Bounds3D.T(meets_before(epsilon=1)) ]) ] results = is1.match(pattern, exact=True) self.assertEqual(len(results), 1) result = results[0] self.assertIntervalsEq(left_box_frame_1, result['left']) self.assertIntervalsEq(right_box_frame_1, result['right']) self.assertIntervalsEq(top_right_box_frame_2, result['top']) self.assertIntervalsEq(bottom_left_box_frame_2, result['bottom'])
def test_minus_self(self): is1 = IntervalSet([ Interval(Bounds3D(2, 2.5)), Interval(Bounds3D(2, 2.7)), Interval(Bounds3D(2.9, 3.5)), Interval(Bounds3D(3.5, 3.6)), Interval(Bounds3D(5, 7)), Interval(Bounds3D(9, 12)), ]) is1 = is1.minus(is1) self.assertIntervalSetEq(is1, IntervalSet([]), eq)
def test_join(self): is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2), ]) is2 = IntervalSet([ Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 4), Interval(Bounds3D(0, 1, 0, 1, 0, 1), 8), ]) is3 = is1.join( is2, Bounds3D.T(overlaps()), lambda i1, i2: Interval( i1['bounds'].intersect_time_span_space(i2['bounds']), i1[ 'payload'] + i2['payload'])) target = IntervalSet([ Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 5), Interval(Bounds3D(0, 1, 0, 1, 0, 1), 9), Interval(Bounds3D(0, 0.5, 0, 1, 0, 1), 10), ]) self.assertIntervalSetEq(is3, target, eq)
def test_map_payload(self): is1 = IntervalSet([ Interval(Bounds3D(0, 1), payload=1), Interval(Bounds3D(2, 3), payload=2), Interval(Bounds3D(4, 5), payload=3), ]) is2 = is1.map_payload(lambda p: p + 10) target = IntervalSet([ Interval(Bounds3D(0, 1), payload=11), Interval(Bounds3D(2, 3), payload=12), Interval(Bounds3D(4, 5), payload=13), ]) self.assertIntervalSetEq(is2, target)
def new_fn_backward(i1: Interval) -> Interval: tracker = cv2.TrackerCSRT_create() ret_bounds = i1.bounds ret_payload = { trajectory_key: [ VideoFrameInterval(i1.bounds, root_decoder=decoder), ] } # buffer all frames in window at once ts_range = list(range(i1['t1'], max(-1, i1['t1'] - window), -step)) # reverse order start_fid = min(ts_range) # inclusive end_fid = max(ts_range) + 1 # exclusive frames_to_track = decoder.get_frame_interval( start_fid, end_fid, step)[::-1] # reverse tracking order # init tracker. For tracking, we must get whole frames H, W = frames_to_track[0].shape[:2] # tracking box in cv2 is the form (x, y, w, h) init_box = np.array( [i1['x1'] * W, i1['y1'] * H, _width(i1) * W, _height(i1) * H]).astype(np.int32) tracker.init(frames_to_track[0], tuple(init_box)) # iterate remaining frames and update tracker, get tracked result for ts, next_frame in zip(ts_range[1:], frames_to_track[1:]): # tracking backward (success, next_box) = tracker.update(next_frame) if success: x, y, w, h = next_box # pixel coord x1, y1, x2, y2 = x, y, x + w, y + h x1, y1, x2, y2 = x1 / W, y1 / H, x2 / W, y2 / H # relative coord next_bounds = Bounds3D(ts, ts + 1, x1, x2, y1, y2) ret_bounds = ret_bounds.span(next_bounds) ret_payload[trajectory_key].insert( 0, VideoFrameInterval(next_bounds, root_decoder=decoder)) else: break return Interval(ret_bounds, ret_payload)
def execute(self): while not self.done and len(self.result_buffer) == 0: i1 = self.instream.get() if i1 is None or i1['t1'] > self.cur_t1 + self.window: # release pending trackings for _, v in self.trackings.items(): # v is a list of (t1, x1, y1, x2, y2) new_trajectory = [] for t1, x1, y1, x2, y2 in v: new_trajectory.append( Interval(Bounds3D(t1, t1 + 1, x1, x2, y1, y2))) new_bounds = functools.reduce( Bounds3D.span, [i.bounds for i in new_trajectory]) new_payload = {self.trajectory_key: new_trajectory} self.result_buffer.append(Interval(new_bounds, new_payload)) self.result_buffer.sort(key=lambda i: i['t1']) self.trackings.clear() if i1 is None: self.done = True break else: # reset tracker and cur_t1 self.tracker = SORTTrack(self.max_age, self.min_hits, self.iou_threshold) self.cur_t1 = i1['t1'] dets = self.tracker.update(self.get_boxes_fn(i1)) assert dets.shape[1] == 5, str(dets) for x1, y1, x2, y2, oid in dets: self.trackings[oid].append([i1['t1'], x1, y1, x2, y2]) if len(self.result_buffer) > 0: self.publish(self.result_buffer.pop(0)) return True else: return False
def test_coalesce_with_pred(self): def overlapping_bboxes(intrvl1, intrvl2): if Bounds3D.X(overlaps())(intrvl1, intrvl2) and Bounds3D.Y( overlaps())(intrvl1, intrvl2): return True else: return False is1 = IntervalSet([ Interval(Bounds3D(2, 5, 0.2, 0.8, 0.2, 0.4), 1), Interval(Bounds3D(1, 10, 0.3, 0.4, 0.3, 0.6), 1), Interval(Bounds3D(9, 11, 0.16, 0.17, 0.3, 0.5), 1), Interval(Bounds3D(13, 15, 0.5, 1, 0, 0.5), 1), Interval(Bounds3D(14, 19, 0.5, 1, 0, 0.5), 1), ]) target = IntervalSet([ Interval(Bounds3D(1, 10, 0.2, 0.8, 0.2, 0.6), 2), Interval(Bounds3D(9, 11, 0.16, 0.17, 0.3, 0.5), 1), Interval(Bounds3D(13, 19, 0.5, 1, 0, 0.5), 2), ]) self.assertIntervalSetEq( is1.coalesce(('t1', 't2'), Bounds3D.span, payload_plus, overlapping_bboxes), target)