def test_group_by(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1)),
Interval(Bounds3D(1, 2)),
Interval(Bounds3D(0, 2)),
Interval(Bounds3D(1, 3)),
])
def merge_intervals(k, intervals):
merged = intervals.fold(
lambda i1, i2: Interval(i1['bounds'].span(i2['bounds'])))
merged['payload'] = intervals
return merged
is2 = is1.group_by(lambda i: int(i['t2']) % 2, merge_intervals)
target = IntervalSet([
Interval(Bounds3D(0, 3),
payload=IntervalSet([
Interval(Bounds3D(0, 1)),
Interval(Bounds3D(1, 3)),
])),
Interval(Bounds3D(0, 2),
payload=IntervalSet([
Interval(Bounds3D(0, 2)),
Interval(Bounds3D(1, 2)),
])),
])
self.assertIntervalSetEq(is2, target,
self.compare_interval_sets_in_payload())
def test_len(self):
is1 = IntervalSet([
Interval(Bounds3D(1, 5)),
Interval(Bounds3D(10, 22)),
])
self.assertEqual(len(is1), 2)
self.assertEqual(is1.size(), 2)
def test_collect_by_interval(self):
c = TestIntervalSetMapping.get_collection()
d = IntervalSetMapping({1: IntervalSet([
Interval(Bounds3D(t,t)) for t in range(1, 100)])})
e = c.collect_by_interval(d, Bounds3D.T(overlaps()),
filter_empty=False, window=0)
self.assertEqual(e.keys(), c.keys())
def test_match_multiple_solutions(self):
left_box_frame_1 = Interval(Bounds3D(1, 1, 0.1, 0.4, 0.4, 0.8))
right_box_frame_1 = Interval(Bounds3D(1, 1, 0.6, 0.9, 0.3, 0.7))
bottom_left_box_frame_2 = Interval(Bounds3D(2, 2, 0.1, 0.3, 0.8, 0.9))
top_right_box_frame_2 = Interval(Bounds3D(2, 2, 0.5, 0.7, 0.2, 0.7))
is1 = IntervalSet([
left_box_frame_1,
right_box_frame_1,
bottom_left_box_frame_2,
top_right_box_frame_2,
Interval(Bounds3D(3, 3)),
])
pattern = [
(
["left", "right"],
[
# Two boxes on left and right on same frame
Bounds3D.T(equal()),
Bounds3D.XY(left_of()),
]),
]
results = is1.match(pattern, exact=False)
self.assertEqual(len(results), 2)
# Add single interval constraints.
pattern = pattern + [
(["left"], [Bounds3D.XY(height_at_least(0.3))]),
(["right"], [Bounds3D.XY(height_at_least(0.3))]),
]
results = is1.match(pattern, exact=False)
self.assertEqual(len(results), 1)
result = results[0]
self.assertIntervalsEq(left_box_frame_1, result['left'])
self.assertIntervalsEq(right_box_frame_1, result['right'])
def test_filter(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
is1 = is1.filter(lambda i: i['payload'] > 2)
target = IntervalSet([
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
self.assertIntervalSetEq(is1, target, eq)
def test_fold(self):
def fold_fn(acc, i):
return acc + (i['bounds'].length() * i['bounds'].width() *
i['bounds'].height())
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
self.assertAlmostEqual(is1.fold(fold_fn, 0), 1.375)
def test_join_with_optimization_window(self):
is1 = IntervalSet(
[Interval(Bounds3D(t, t + 1), t) for t in range(100)])
is2 = IntervalSet([Interval(Bounds3D(t, t), t) for t in range(100)])
is3 = is1.join(is2,
Bounds3D.T(before(max_dist=1)),
lambda i1, i2: Interval(i1['bounds'].span(i2['bounds']),
i2['payload']),
window=1)
target = IntervalSet(
[Interval(Bounds3D(t, t + 2), t + 2) for t in range(98)] +
[Interval(Bounds3D(t, t + 1), t + 1) for t in range(99)])
self.assertIntervalSetEq(is3, target, eq)
def test_fold_custom_sortkey(self):
def sortkey(i):
return (i['x1'], i['x2'])
def fold_fn(acc, i):
acc.append(i['payload'])
return acc
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 0.1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0.1, 0.4, 0, 1, 0, 0.5), 3),
])
self.assertListEqual(is1.fold(fold_fn, [], sortkey), [1, 3, 2])
def list_to_IntervalSetMapping(interval_list):
ism = {}
for video_id, start, end, duration in interval_list:
if not video_id in ism:
ism[video_id] = []
ism[video_id].append(Interval(Bounds3D(start, end)))
return IntervalSetMapping({video_id: IntervalSet(intervalSet) for video_id, intervalSet in ism.items()})
def flatten_fn(intrvl):
rv = []
try:
detections = intrvl.payload[self.result_key]
except KeyError:
raise KeyError(
f"Cannot find {self.result_key} in input payload. Did you run object detection on the input stream?"
)
for box, score, class_name in zip(detections['detection_boxes'],
detections['detection_scores'],
detections['detection_names']):
if score < self.confidence:
break
if (not self.black_list and class_name in self.targets) or (
self.black_list and class_name not in self.targets):
# create new patch
top, left, bottom, right = box # TF return between 0~1
# the following arithmetic should be correct even if `intrvl` is not full frame.
new_bounds = Bounds3D(
intrvl['t1'], intrvl['t2'],
intrvl['x1'] + intrvl.bounds.width() * left,
intrvl['x1'] + intrvl.bounds.width() * right,
intrvl['y1'] + intrvl.bounds.height() * top,
intrvl['y1'] + intrvl.bounds.height() * bottom)
new_patch = ImageInterval(new_bounds, root=intrvl.root)
rv.append(new_patch)
rv.sort(key=lambda i: i.bounds)
return rv
def dilate_car(icar: Interval) -> Interval:
carh, carw = _height(icar), _width(icar)
new_bounds = Bounds3D(t1=int(max(0, icar['t1'] - fps)),
t2=int(min(frame_count, icar['t2'] + fps)),
x1=max(0, icar['x1'] - carw),
x2=min(1, icar['x2'] + carw),
y1=max(0, icar['y1'] - carh),
y2=min(1, icar['y2'] + carh))
return Interval(new_bounds)
def split_fn(i):
output = []
t = i.copy()
while t['bounds'].length() > 5:
output.append(Interval(Bounds3D(t['t1'], t['t1'] + 5)))
t['t1'] = t['t1'] + 5
if t['bounds'].length() > 0:
output.append(t)
return IntervalSet(output)
def average_space_span_time(list_of_box):
ret_bounds = Bounds3D(
t1=min([b['t1'] for b in list_of_box]),
t2=max([b['t2'] for b in list_of_box]),
)
for key in ('x1', 'x2', 'y1', 'y2'):
ret_bounds[key] = np.mean([b[key] for b in list_of_box])
return ret_bounds
def test_map(self):
def expand_to_frame(intrvl):
new_bounds = intrvl['bounds'].copy()
new_bounds['x1'] = 0
new_bounds['x2'] = 1
new_bounds['y1'] = 0
new_bounds['y2'] = 1
return Interval(new_bounds, intrvl['payload'])
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0.3, 0.4, 0.5, 0.6)),
Interval(Bounds3D(0, 0.5, 0.2, 0.3, 0.5, 0.6))
])
is1 = is1.map(expand_to_frame)
target = IntervalSet([
Interval(Bounds3D(0, 0.5, 0, 1, 0, 1)),
Interval(Bounds3D(0, 1, 0, 1, 0, 1)),
])
self.assertIntervalSetEq(is1, target)
def bounds_parser(item):
args = [
key_accessor(item, schema_final['t1']),
key_accessor(item, schema_final['t2'])
]
kwargs = {}
for k in ['x1', 'x2', 'y1', 'y2']:
if k in schema_final:
kwargs[k] = key_accessor(item, schema_final[k])
return Bounds3D(*args, **kwargs)
def dilate_op(i:ImageInterval) -> ImageInterval:
w, h = _width(i), _height(i)
new_bounds = Bounds3D(
i['t1'], i['t2'],
x1=max(0, i['x1']-w*3),
x2=min(1, i['x2']+w*3),
y1=max(0, i['y1']-h/2),
y2=min(1, i['y2']+h/2)
)
return ImageInterval(new_bounds, None, root=i.root)
def test_minus_everything(self):
is1 = IntervalSet(
[Interval(Bounds3D(1, 10)),
Interval(Bounds3D(3, 15))])
is2 = IntervalSet([
Interval(Bounds3D(2, 2.5)),
Interval(Bounds3D(2, 2.7)),
Interval(Bounds3D(2.9, 3.5)),
Interval(Bounds3D(3.5, 3.6)),
Interval(Bounds3D(5, 7)),
Interval(Bounds3D(9, 12)),
])
is3 = is2.minus(is1)
self.assertIntervalSetEq(is3, IntervalSet([]), eq)
def test_minus_with_single_frame(self):
is1 = IntervalSet([
Interval(Bounds3D(1, 1)),
Interval(Bounds3D(3, 3)),
Interval(Bounds3D(4, 4)),
Interval(Bounds3D(7, 7)),
Interval(Bounds3D(10, 10)),
])
is2 = IntervalSet([
Interval(Bounds3D(1, 3)),
Interval(Bounds3D(5, 8)),
Interval(Bounds3D(9, 9)),
])
is3 = is1.minus(is2)
target = IntervalSet([
Interval(Bounds3D(4, 4)),
Interval(Bounds3D(10, 10)),
])
self.assertIntervalSetEq(is3, target)
is4 = is2.minus(is1)
self.assertIntervalSetEq(is4, is2)
def test_minus_against_nothing(self):
is1 = IntervalSet([
Interval(Bounds3D(1, 10, 0, 0.5, 0.2, 0.8), 1),
Interval(Bounds3D(3, 15, 0, 1, 0, 1), 2)
])
is2 = IntervalSet([
Interval(Bounds3D(20, 20.5)),
Interval(Bounds3D(20, 20.7)),
Interval(Bounds3D(20.9, 23.5)),
Interval(Bounds3D(23.5, 23.6)),
Interval(Bounds3D(25, 27)),
Interval(Bounds3D(29, 32)),
])
is3 = is1.minus(is2)
self.assertIntervalSetEq(is3, is1, eq)
def test_filter_against(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1)),
Interval(Bounds3D(2, 5)),
Interval(Bounds3D(6, 7)),
])
is2 = IntervalSet([
Interval(Bounds3D(1, 1)),
Interval(Bounds3D(7, 7)),
])
# Take only intervals in is1 that overlaps with some interval in is2
is3 = is1.filter_against(is2, Bounds3D.T(overlaps()), window=0)
self.assertIntervalSetEq(
is3,
IntervalSet([Interval(Bounds3D(0, 1)),
Interval(Bounds3D(6, 7))]))
def test_group_by_axis(self):
default_bounds = Bounds3D(0, 1, 0, 1, 0, 1)
intervals_1 = [
Interval(Bounds3D(1, 1, 0.4, 0.5, 0.6, 0.8), 1),
Interval(Bounds3D(1, 1, 0.1, 0.2, 0.2, 0.3), 2),
Interval(Bounds3D(1, 1, 0.3, 0.5, 0.1, 0.5), 3),
]
intervals_2 = [
Interval(Bounds3D(2, 2, 0.3, 0.5, 0.6, 0.8), 11),
Interval(Bounds3D(2, 2, 0.2, 0.3, 0.2, 0.9), 12),
Interval(Bounds3D(2, 2, 0.3, 0.7, 0, 0.5), 13),
]
is1 = IntervalSet(intervals_1 + intervals_2)
target = IntervalSet([
Interval(Bounds3D(1, 1), payload=IntervalSet(intervals_1)),
Interval(Bounds3D(2, 2), payload=IntervalSet(intervals_2)),
])
self.assertIntervalSetEq(
is1.group_by_axis(('t1', 't2'), default_bounds), target,
self.compare_interval_sets_in_payload())
def merge_candidate(iperson: Interval, icar: Interval) -> Interval:
carh, carw = _height(icar), _width(icar)
new_bounds = Bounds3D(t1=int(max(0, icar['t1'] - fps * 3)),
t2=int(max(0, iperson['t1'] + fps * 3)),
x1=max(0, icar['x1'] - carw),
x2=min(1, icar['x2'] + carw),
y1=max(0, icar['y1'] - carh),
y2=min(1, icar['y2'] + carh))
new_payload = {
'traj_person': iperson.payload['traj_person'],
'stopped_car': icar
}
return Interval(new_bounds, new_payload)
def test_union(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 1),
])
is2 = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 2),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 2),
])
is3 = is1.union(is2)
target = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 2),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 1),
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 2),
])
self.assertIntervalSetEq(is3, target, eq)
def test_match(self):
left_box_frame_1 = Interval(Bounds3D(1, 1, 0.1, 0.4, 0.4, 0.8))
right_box_frame_1 = Interval(Bounds3D(1, 1, 0.6, 0.9, 0.3, 0.7))
bottom_left_box_frame_2 = Interval(Bounds3D(2, 2, 0.1, 0.3, 0.8, 0.9))
top_right_box_frame_2 = Interval(Bounds3D(2, 2, 0.5, 0.7, 0.2, 0.7))
is1 = IntervalSet([
left_box_frame_1, right_box_frame_1, bottom_left_box_frame_2,
top_right_box_frame_2
])
pattern = [
(
["left", "right"],
[
# Two boxes on left and right on same frame
Bounds3D.T(equal()),
Bounds3D.XY(left_of()),
]),
(
["top", "bottom"],
[
# Two boxes on top and bottom on overlapping frame
Bounds3D.T(equal()),
Bounds3D.XY(above()),
]),
(
["left", "top"],
[
# Left-Right pattern comes before Top-Bottom
Bounds3D.T(meets_before(epsilon=1))
])
]
results = is1.match(pattern, exact=True)
self.assertEqual(len(results), 1)
result = results[0]
self.assertIntervalsEq(left_box_frame_1, result['left'])
self.assertIntervalsEq(right_box_frame_1, result['right'])
self.assertIntervalsEq(top_right_box_frame_2, result['top'])
self.assertIntervalsEq(bottom_left_box_frame_2, result['bottom'])
def test_minus_self(self):
is1 = IntervalSet([
Interval(Bounds3D(2, 2.5)),
Interval(Bounds3D(2, 2.7)),
Interval(Bounds3D(2.9, 3.5)),
Interval(Bounds3D(3.5, 3.6)),
Interval(Bounds3D(5, 7)),
Interval(Bounds3D(9, 12)),
])
is1 = is1.minus(is1)
self.assertIntervalSetEq(is1, IntervalSet([]), eq)
def test_join(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
])
is2 = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 4),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 8),
])
is3 = is1.join(
is2, Bounds3D.T(overlaps()), lambda i1, i2: Interval(
i1['bounds'].intersect_time_span_space(i2['bounds']), i1[
'payload'] + i2['payload']))
target = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 5),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 9),
Interval(Bounds3D(0, 0.5, 0, 1, 0, 1), 10),
])
self.assertIntervalSetEq(is3, target, eq)
def test_map_payload(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1), payload=1),
Interval(Bounds3D(2, 3), payload=2),
Interval(Bounds3D(4, 5), payload=3),
])
is2 = is1.map_payload(lambda p: p + 10)
target = IntervalSet([
Interval(Bounds3D(0, 1), payload=11),
Interval(Bounds3D(2, 3), payload=12),
Interval(Bounds3D(4, 5), payload=13),
])
self.assertIntervalSetEq(is2, target)
def new_fn_backward(i1: Interval) -> Interval:
tracker = cv2.TrackerCSRT_create()
ret_bounds = i1.bounds
ret_payload = {
trajectory_key: [
VideoFrameInterval(i1.bounds, root_decoder=decoder),
]
}
# buffer all frames in window at once
ts_range = list(range(i1['t1'], max(-1, i1['t1'] - window),
-step)) # reverse order
start_fid = min(ts_range) # inclusive
end_fid = max(ts_range) + 1 # exclusive
frames_to_track = decoder.get_frame_interval(
start_fid, end_fid, step)[::-1] # reverse tracking order
# init tracker. For tracking, we must get whole frames
H, W = frames_to_track[0].shape[:2]
# tracking box in cv2 is the form (x, y, w, h)
init_box = np.array(
[i1['x1'] * W, i1['y1'] * H,
_width(i1) * W,
_height(i1) * H]).astype(np.int32)
tracker.init(frames_to_track[0], tuple(init_box))
# iterate remaining frames and update tracker, get tracked result
for ts, next_frame in zip(ts_range[1:], frames_to_track[1:]):
# tracking backward
(success, next_box) = tracker.update(next_frame)
if success:
x, y, w, h = next_box # pixel coord
x1, y1, x2, y2 = x, y, x + w, y + h
x1, y1, x2, y2 = x1 / W, y1 / H, x2 / W, y2 / H # relative coord
next_bounds = Bounds3D(ts, ts + 1, x1, x2, y1, y2)
ret_bounds = ret_bounds.span(next_bounds)
ret_payload[trajectory_key].insert(
0, VideoFrameInterval(next_bounds, root_decoder=decoder))
else:
break
return Interval(ret_bounds, ret_payload)
def execute(self):
while not self.done and len(self.result_buffer) == 0:
i1 = self.instream.get()
if i1 is None or i1['t1'] > self.cur_t1 + self.window:
# release pending trackings
for _, v in self.trackings.items():
# v is a list of (t1, x1, y1, x2, y2)
new_trajectory = []
for t1, x1, y1, x2, y2 in v:
new_trajectory.append(
Interval(Bounds3D(t1, t1 + 1, x1, x2, y1, y2)))
new_bounds = functools.reduce(
Bounds3D.span, [i.bounds for i in new_trajectory])
new_payload = {self.trajectory_key: new_trajectory}
self.result_buffer.append(Interval(new_bounds,
new_payload))
self.result_buffer.sort(key=lambda i: i['t1'])
self.trackings.clear()
if i1 is None:
self.done = True
break
else:
# reset tracker and cur_t1
self.tracker = SORTTrack(self.max_age, self.min_hits,
self.iou_threshold)
self.cur_t1 = i1['t1']
dets = self.tracker.update(self.get_boxes_fn(i1))
assert dets.shape[1] == 5, str(dets)
for x1, y1, x2, y2, oid in dets:
self.trackings[oid].append([i1['t1'], x1, y1, x2, y2])
if len(self.result_buffer) > 0:
self.publish(self.result_buffer.pop(0))
return True
else:
return False
def test_coalesce_with_pred(self):
def overlapping_bboxes(intrvl1, intrvl2):
if Bounds3D.X(overlaps())(intrvl1, intrvl2) and Bounds3D.Y(
overlaps())(intrvl1, intrvl2):
return True
else:
return False
is1 = IntervalSet([
Interval(Bounds3D(2, 5, 0.2, 0.8, 0.2, 0.4), 1),
Interval(Bounds3D(1, 10, 0.3, 0.4, 0.3, 0.6), 1),
Interval(Bounds3D(9, 11, 0.16, 0.17, 0.3, 0.5), 1),
Interval(Bounds3D(13, 15, 0.5, 1, 0, 0.5), 1),
Interval(Bounds3D(14, 19, 0.5, 1, 0, 0.5), 1),
])
target = IntervalSet([
Interval(Bounds3D(1, 10, 0.2, 0.8, 0.2, 0.6), 2),
Interval(Bounds3D(9, 11, 0.16, 0.17, 0.3, 0.5), 1),
Interval(Bounds3D(13, 19, 0.5, 1, 0, 0.5), 2),
])
self.assertIntervalSetEq(
is1.coalesce(('t1', 't2'), Bounds3D.span, payload_plus,
overlapping_bboxes), target)
