def test_match_multiple_solutions(self):
left_box_frame_1 = Interval(Bounds3D(1, 1, 0.1, 0.4, 0.4, 0.8))
right_box_frame_1 = Interval(Bounds3D(1, 1, 0.6, 0.9, 0.3, 0.7))
bottom_left_box_frame_2 = Interval(Bounds3D(2, 2, 0.1, 0.3, 0.8, 0.9))
top_right_box_frame_2 = Interval(Bounds3D(2, 2, 0.5, 0.7, 0.2, 0.7))
is1 = IntervalSet([
left_box_frame_1,
right_box_frame_1,
bottom_left_box_frame_2,
top_right_box_frame_2,
Interval(Bounds3D(3, 3)),
])
pattern = [
(
["left", "right"],
[
# Two boxes on left and right on same frame
Bounds3D.T(equal()),
Bounds3D.XY(left_of()),
]),
]
results = is1.match(pattern, exact=False)
self.assertEqual(len(results), 2)
# Add single interval constraints.
pattern = pattern + [
(["left"], [Bounds3D.XY(height_at_least(0.3))]),
(["right"], [Bounds3D.XY(height_at_least(0.3))]),
]
results = is1.match(pattern, exact=False)
self.assertEqual(len(results), 1)
result = results[0]
self.assertIntervalsEq(left_box_frame_1, result['left'])
self.assertIntervalsEq(right_box_frame_1, result['right'])
def test_group_by(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1)),
Interval(Bounds3D(1, 2)),
Interval(Bounds3D(0, 2)),
Interval(Bounds3D(1, 3)),
])
def merge_intervals(k, intervals):
merged = intervals.fold(
lambda i1, i2: Interval(i1['bounds'].span(i2['bounds'])))
merged['payload'] = intervals
return merged
is2 = is1.group_by(lambda i: int(i['t2']) % 2, merge_intervals)
target = IntervalSet([
Interval(Bounds3D(0, 3),
payload=IntervalSet([
Interval(Bounds3D(0, 1)),
Interval(Bounds3D(1, 3)),
])),
Interval(Bounds3D(0, 2),
payload=IntervalSet([
Interval(Bounds3D(0, 2)),
Interval(Bounds3D(1, 2)),
])),
])
self.assertIntervalSetEq(is2, target,
self.compare_interval_sets_in_payload())
def test_len(self):
is1 = IntervalSet([
Interval(Bounds3D(1, 5)),
Interval(Bounds3D(10, 22)),
])
self.assertEqual(len(is1), 2)
self.assertEqual(is1.size(), 2)
def process_single(black_frames_path, captions_path, metadata_path,
commercials_outpath):
# Load original data
black_frames = load_json(black_frames_path)
captions = load_captions(captions_path)
metadata = load_json(metadata_path)
# Create IntervalSet objects
black_frames_set = IntervalSet([
Interval(
Bounds3D(frame_num / metadata['fps'],
(frame_num + 1) / metadata['fps']))
for frame_num in black_frames
])
captions_set = IntervalSet([
Interval(Bounds3D(start, end), payload=text)
for text, start, end, in captions
])
whole_video = IntervalSet(
[Interval(Bounds3D(0, metadata['frames'] / metadata['fps']))])
# Detect commercials
results = detect_commercials(black_frames_set, captions_set, whole_video)
# Convert commercial intervals back to frames
results = convert_set_from_seconds_to_frames(results, metadata['fps'])
# Save results in JSON format
results = [(r['t1'], r['t2']) for r in results.get_intervals()]
save_json(results, commercials_outpath)
def fold_fn(stack, interval):
if interval['t2'] - interval['t1'] > MAX_COMMERCIAL_TIME:
interval = Interval(
Bounds3D(interval['t1'], interval['t1'] + MAX_COMMERCIAL_TIME))
if len(stack) == 0:
stack.append(interval)
else:
last = stack.pop()
if or_pred(overlaps(),
after(max_dist=COMMERCIAL_FOLD_EPSILON))(interval,
last):
if last['bounds'].span(interval['bounds']).size() \
> MAX_COMMERCIAL_TIME:
stack.append(
Interval(
Bounds3D(last['t1'],
last['t1'] + MAX_COMMERCIAL_TIME)))
else:
stack.append(
Interval(last['bounds'].span(interval['bounds'])))
else:
stack.append(last)
stack.append(interval)
return stack
def main():
'''
An example of a join. We filter pairs based on time overlap, and merge
surviving pairs by taking the intersection over the time bounds and the
span of the spatial bounds (and adding the payloads).
'''
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
])
is2 = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 4),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 8),
])
is3 = is1.join(
is2, Bounds3D.T(overlaps()), lambda i1, i2: Interval(
i1['bounds'].intersect_time_span_space(i2['bounds']), i1['payload']
+ i2['payload']))
print('is1:')
print(is1)
print('is2:')
print(is2)
print('is1 joined with is2:')
print(is3)
def test_fold(self):
def fold_fn(acc, i):
return acc + (i['bounds'].length() * i['bounds'].width() *
i['bounds'].height())
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
self.assertAlmostEqual(is1.fold(fold_fn, 0), 1.375)
def test_filter(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
is1 = is1.filter(lambda i: i['payload'] > 2)
target = IntervalSet([
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
self.assertIntervalSetEq(is1, target, eq)
def test_join_with_optimization_window(self):
is1 = IntervalSet(
[Interval(Bounds3D(t, t + 1), t) for t in range(100)])
is2 = IntervalSet([Interval(Bounds3D(t, t), t) for t in range(100)])
is3 = is1.join(is2,
Bounds3D.T(before(max_dist=1)),
lambda i1, i2: Interval(i1['bounds'].span(i2['bounds']),
i2['payload']),
window=1)
target = IntervalSet(
[Interval(Bounds3D(t, t + 2), t + 2) for t in range(98)] +
[Interval(Bounds3D(t, t + 1), t + 1) for t in range(99)])
self.assertIntervalSetEq(is3, target, eq)
def test_fold_custom_sortkey(self):
def sortkey(i):
return (i['x1'], i['x2'])
def fold_fn(acc, i):
acc.append(i['payload'])
return acc
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 0.1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0.1, 0.4, 0, 1, 0, 0.5), 3),
])
self.assertListEqual(is1.fold(fold_fn, [], sortkey), [1, 3, 2])
def expand_to_frame(intrvl):
new_bounds = intrvl['bounds'].copy()
new_bounds['x1'] = 0
new_bounds['x2'] = 1
new_bounds['y1'] = 0
new_bounds['y2'] = 1
return Interval(new_bounds, intrvl['payload'])
def test_collect_by_interval(self):
c = TestIntervalSetMapping.get_collection()
d = IntervalSetMapping({1: IntervalSet([
Interval(Bounds3D(t,t)) for t in range(1, 100)])})
e = c.collect_by_interval(d, Bounds3D.T(overlaps()),
filter_empty=False, window=0)
self.assertEqual(e.keys(), c.keys())
def find_clips_for_keyword(keyword, use_only_video=False):
ism = {}
for vm in tqdm(video_metadata):
if not vm["annotation_filename"] or (not vm["only_video"]
and use_only_video):
continue
try:
h5 = eeghdf.Eeghdf(vm["annotation_filename"])
except:
print(vm["annotation_filename"])
os.remove(vm["annotation_filename"])
continue
starts = [start / 10**7 for start in h5._annotation_start100ns]
texts = h5._annotation_text
if not keyword or any(keyword.lower() in text.lower()
for text in texts):
interval_set = IntervalSet([
Interval(
Bounds3D(start, start + 5), # we set the duration
{
'spatial_type':
SpatialType_Caption(">>" + text + "\n"),
'metadata': {}
}) for start, text in zip(starts, texts)
])
ism[vm["id"]] = interval_set
print(
f"Found {len(ism)} videos with keyword {keyword} in the annotations.")
vgrid_spec = VGridSpec(video_meta=video_metadata_wrapper,
vis_format=VideoBlockFormat(imaps=[('bboxes',
ism)]),
video_endpoint='http://localhost:8080')
return VGridWidget(vgrid_spec=vgrid_spec.to_json_compressed())
def list_to_IntervalSetMapping(interval_list):
ism = {}
for video_id, start, end, duration in interval_list:
if not video_id in ism:
ism[video_id] = []
ism[video_id].append(Interval(Bounds3D(start, end)))
return IntervalSetMapping({video_id: IntervalSet(intervalSet) for video_id, intervalSet in ism.items()})
def get_ground_truth():
interval = 30
GT_FOLDER = 'empty_spaces'
empty_parking_spaces = [
pickle.loads(
requests.get(posixpath.join(
posixpath.join(VIDEO_COLLECTION_BASEURL, GT_FOLDER),
posixpath.join(vm.path[:-4], 'gt.pkl')),
verify=False).content) for vm in video_metadata
]
gt_ism = IntervalSetMapping({
metadata.id: IntervalSet([
Interval(
Bounds3D(t1=30 * i / metadata.fps,
t2=30 * (i + interval) / metadata.fps,
x1=bbox[0] / metadata.width + .01,
x2=bbox[2] / metadata.width - .01,
y1=bbox[1] / metadata.height + .01,
y2=bbox[3] / metadata.height - .01))
for i, frame in enumerate(space_frame_list) if (i % interval == 0)
for bbox in frame
])
for space_frame_list, metadata in tqdm(zip(empty_parking_spaces,
video_metadata),
total=len(empty_parking_spaces))
})
return gt_ism
def get_maskrcnn_bboxes():
interval = 30
bboxes = [
pickle.loads(
requests.get(posixpath.join(
posixpath.join(VIDEO_COLLECTION_BASEURL, BBOX_FOLDER),
posixpath.join(vm.path[:-4], 'bboxes.pkl')),
verify=False).content) for vm in (video_metadata)
]
bboxes_ism = IntervalSetMapping({
metadata.id: IntervalSet([
Interval(Bounds3D(t1=30 * i / metadata.fps,
t2=30 * (i + interval) / metadata.fps,
x1=bbox[0] / metadata.width,
x2=bbox[2] / metadata.width,
y1=bbox[1] / metadata.height,
y2=bbox[3] / metadata.height),
payload={
'class': bbox[4],
'score': bbox[5],
'spatial_type': SpatialType_Bbox(text=bbox[4])
}) for i, frame in enumerate(bbox_frame_list)
if (i % interval == 0) for bbox in frame
])
for bbox_frame_list, metadata in tqdm(zip(bboxes, (video_metadata)),
total=len(bboxes))
})
return bboxes_ism
def IntervalSetMapping_second_to_frame(ism):
intervalSets_frame = {}
for video_id, intervalSet in ism.get_grouped_intervals().items():
video = Video.objects.filter(id=video_id)[0]
fps = video.fps
intervalSets_frame[video_id] = IntervalSet(
[Interval(Bounds1D(int(i.bounds['t1'] * fps), int(i.bounds['t2'] * fps)), i.payload) \
for i in intervalSet.get_intervals()] )
return IntervalSetMapping(intervalSets_frame)
def split_fn(i):
output = []
t = i.copy()
while t['bounds'].length() > 5:
output.append(Interval(Bounds3D(t['t1'], t['t1'] + 5)))
t['t1'] = t['t1'] + 5
if t['bounds'].length() > 0:
output.append(t)
return IntervalSet(output)
def test_map(self):
def expand_to_frame(intrvl):
new_bounds = intrvl['bounds'].copy()
new_bounds['x1'] = 0
new_bounds['x2'] = 1
new_bounds['y1'] = 0
new_bounds['y2'] = 1
return Interval(new_bounds, intrvl['payload'])
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0.3, 0.4, 0.5, 0.6)),
Interval(Bounds3D(0, 0.5, 0.2, 0.3, 0.5, 0.6))
])
is1 = is1.map(expand_to_frame)
target = IntervalSet([
Interval(Bounds3D(0, 0.5, 0, 1, 0, 1)),
Interval(Bounds3D(0, 1, 0, 1, 0, 1)),
])
self.assertIntervalSetEq(is1, target)
def rs_to_rekall(rs_ilist, video_ids=None, with_payload=True):
rekall_ism = {}
if video_ids is None:
video_ids = rs_ilist.get_ids()
for video_id in video_ids:
if with_payload:
interval_list = rs_ilist.get_intervals_with_payload(video_id, True)
rekall_ism[video_id] = IntervalSet([
Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0),
i[2]) for i in interval_list
])
else:
interval_list = rs_ilist.get_intervals(video_id, True)
rekall_ism[video_id] = IntervalSet([
Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0))
for i in interval_list
])
return IntervalSetMapping(rekall_ism)
def for_vgrid(self):
fps = self.obj.video.fps
pad = 0.02
return IntervalSet([
Interval(
Bounds3D(t1=p.frame / fps,
t2=(p.frame + 1) / fps,
x1=p.to_pixel_norm()[0] - pad,
x2=p.to_pixel_norm()[0] + pad,
y1=p.to_pixel_norm()[1] - pad,
y2=p.to_pixel_norm()[1] + pad)) for p in self.pos
])
def test_minus_everything(self):
is1 = IntervalSet(
[Interval(Bounds3D(1, 10)),
Interval(Bounds3D(3, 15))])
is2 = IntervalSet([
Interval(Bounds3D(2, 2.5)),
Interval(Bounds3D(2, 2.7)),
Interval(Bounds3D(2.9, 3.5)),
Interval(Bounds3D(3.5, 3.6)),
Interval(Bounds3D(5, 7)),
Interval(Bounds3D(9, 12)),
])
is3 = is2.minus(is1)
self.assertIntervalSetEq(is3, IntervalSet([]), eq)
def __init__(self, block_label_state):
self._new_intervals = IntervalSet([
Interval(
Bounds3D(t1=intvl['bounds']['t1'],
t2=intvl['bounds']['t2'],
x1=intvl['bounds']['bbox']['x1'],
x2=intvl['bounds']['bbox']['x2'],
y1=intvl['bounds']['bbox']['y1'],
y2=intvl['bounds']['bbox']['y2']))
for intvl in block_label_state['new_intervals']
])
self._captions_selected = None # TODO
def test_minus_with_single_frame(self):
is1 = IntervalSet([
Interval(Bounds3D(1, 1)),
Interval(Bounds3D(3, 3)),
Interval(Bounds3D(4, 4)),
Interval(Bounds3D(7, 7)),
Interval(Bounds3D(10, 10)),
])
is2 = IntervalSet([
Interval(Bounds3D(1, 3)),
Interval(Bounds3D(5, 8)),
Interval(Bounds3D(9, 9)),
])
is3 = is1.minus(is2)
target = IntervalSet([
Interval(Bounds3D(4, 4)),
Interval(Bounds3D(10, 10)),
])
self.assertIntervalSetEq(is3, target)
is4 = is2.minus(is1)
self.assertIntervalSetEq(is4, is2)
def test_match(self):
left_box_frame_1 = Interval(Bounds3D(1, 1, 0.1, 0.4, 0.4, 0.8))
right_box_frame_1 = Interval(Bounds3D(1, 1, 0.6, 0.9, 0.3, 0.7))
bottom_left_box_frame_2 = Interval(Bounds3D(2, 2, 0.1, 0.3, 0.8, 0.9))
top_right_box_frame_2 = Interval(Bounds3D(2, 2, 0.5, 0.7, 0.2, 0.7))
is1 = IntervalSet([
left_box_frame_1, right_box_frame_1, bottom_left_box_frame_2,
top_right_box_frame_2
])
pattern = [
(
["left", "right"],
[
# Two boxes on left and right on same frame
Bounds3D.T(equal()),
Bounds3D.XY(left_of()),
]),
(
["top", "bottom"],
[
# Two boxes on top and bottom on overlapping frame
Bounds3D.T(equal()),
Bounds3D.XY(above()),
]),
(
["left", "top"],
[
# Left-Right pattern comes before Top-Bottom
Bounds3D.T(meets_before(epsilon=1))
])
]
results = is1.match(pattern, exact=True)
self.assertEqual(len(results), 1)
result = results[0]
self.assertIntervalsEq(left_box_frame_1, result['left'])
self.assertIntervalsEq(right_box_frame_1, result['right'])
self.assertIntervalsEq(top_right_box_frame_2, result['top'])
self.assertIntervalsEq(bottom_left_box_frame_2, result['bottom'])
def test_minus_against_nothing(self):
is1 = IntervalSet([
Interval(Bounds3D(1, 10, 0, 0.5, 0.2, 0.8), 1),
Interval(Bounds3D(3, 15, 0, 1, 0, 1), 2)
])
is2 = IntervalSet([
Interval(Bounds3D(20, 20.5)),
Interval(Bounds3D(20, 20.7)),
Interval(Bounds3D(20.9, 23.5)),
Interval(Bounds3D(23.5, 23.6)),
Interval(Bounds3D(25, 27)),
Interval(Bounds3D(29, 32)),
])
is3 = is1.minus(is2)
self.assertIntervalSetEq(is3, is1, eq)
def main():
'''
Examples of creating some Intervals. Intervals are parameterized by Bounds
(Bounds3D in these examples) and an optional payload.
'''
my_first_bounds = Bounds3D(t1=5, t2=10, x1=0.1, x2=0.4, y1=0.3, y2=0.9)
# This bounds is the same as my_first_bounds
my_second_bounds = Bounds3D(5, 10, 0.1, 0.4, 0.3, 0.9)
# This Interval has the 3D bounds of my_first_bounds and a payload
my_first_interval = Interval(my_first_bounds, payload=10)
# This Interval doesn't have a payload
my_first_interval_no_payload = Interval(my_first_bounds)
# IntervalSet contains multiple Intervals
my_first_interval_set = IntervalSet([
my_first_interval,
Interval(my_second_bounds, payload='payloads are arbitrary'),
Interval(Bounds3D(20, 30), payload='Bounds3D has default X/Y fields')
])
print(my_first_interval_set)
def main():
'''
Examples of simple functions on IntervalSets.
'''
# filter examples
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
print('is1:')
print(is1)
is1_filtered = is1.filter(lambda i: i['payload'] > 2)
print('is1 filtered on payload:')
print(is1_filtered)
print()
# map example
def expand_to_frame(intrvl):
new_bounds = intrvl['bounds'].copy()
new_bounds['x1'] = 0
new_bounds['x2'] = 1
new_bounds['y1'] = 0
new_bounds['y2'] = 1
return Interval(new_bounds, intrvl['payload'])
is2 = IntervalSet([
Interval(Bounds3D(0, 1, 0.3, 0.4, 0.5, 0.6)),
Interval(Bounds3D(0, 0.5, 0.2, 0.3, 0.5, 0.6))
])
print('is2:')
print(is2)
is2_expanded = is2.map(expand_to_frame)
print('is2 expanded to the frame:')
print(is2_expanded)
def test_union(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 1),
])
is2 = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 2),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 2),
])
is3 = is1.union(is2)
target = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 2),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 1),
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 2),
])
self.assertIntervalSetEq(is3, target, eq)
def test_minus_self(self):
is1 = IntervalSet([
Interval(Bounds3D(2, 2.5)),
Interval(Bounds3D(2, 2.7)),
Interval(Bounds3D(2.9, 3.5)),
Interval(Bounds3D(3.5, 3.6)),
Interval(Bounds3D(5, 7)),
Interval(Bounds3D(9, 12)),
])
is1 = is1.minus(is1)
self.assertIntervalSetEq(is1, IntervalSet([]), eq)