def process_single(black_frames_path, captions_path, metadata_path,
commercials_outpath):
# Load original data
black_frames = load_json(black_frames_path)
captions = load_captions(captions_path)
metadata = load_json(metadata_path)
# Create IntervalSet objects
black_frames_set = IntervalSet([
Interval(
Bounds3D(frame_num / metadata['fps'],
(frame_num + 1) / metadata['fps']))
for frame_num in black_frames
])
captions_set = IntervalSet([
Interval(Bounds3D(start, end), payload=text)
for text, start, end, in captions
])
whole_video = IntervalSet(
[Interval(Bounds3D(0, metadata['frames'] / metadata['fps']))])
# Detect commercials
results = detect_commercials(black_frames_set, captions_set, whole_video)
# Convert commercial intervals back to frames
results = convert_set_from_seconds_to_frames(results, metadata['fps'])
# Save results in JSON format
results = [(r['t1'], r['t2']) for r in results.get_intervals()]
save_json(results, commercials_outpath)
def test_match_multiple_solutions(self):
left_box_frame_1 = Interval(Bounds3D(1, 1, 0.1, 0.4, 0.4, 0.8))
right_box_frame_1 = Interval(Bounds3D(1, 1, 0.6, 0.9, 0.3, 0.7))
bottom_left_box_frame_2 = Interval(Bounds3D(2, 2, 0.1, 0.3, 0.8, 0.9))
top_right_box_frame_2 = Interval(Bounds3D(2, 2, 0.5, 0.7, 0.2, 0.7))
is1 = IntervalSet([
left_box_frame_1,
right_box_frame_1,
bottom_left_box_frame_2,
top_right_box_frame_2,
Interval(Bounds3D(3, 3)),
])
pattern = [
(
["left", "right"],
[
# Two boxes on left and right on same frame
Bounds3D.T(equal()),
Bounds3D.XY(left_of()),
]),
]
results = is1.match(pattern, exact=False)
self.assertEqual(len(results), 2)
# Add single interval constraints.
pattern = pattern + [
(["left"], [Bounds3D.XY(height_at_least(0.3))]),
(["right"], [Bounds3D.XY(height_at_least(0.3))]),
]
results = is1.match(pattern, exact=False)
self.assertEqual(len(results), 1)
result = results[0]
self.assertIntervalsEq(left_box_frame_1, result['left'])
self.assertIntervalsEq(right_box_frame_1, result['right'])
def main():
'''
An example of a join. We filter pairs based on time overlap, and merge
surviving pairs by taking the intersection over the time bounds and the
span of the spatial bounds (and adding the payloads).
'''
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
])
is2 = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 4),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 8),
])
is3 = is1.join(
is2, Bounds3D.T(overlaps()), lambda i1, i2: Interval(
i1['bounds'].intersect_time_span_space(i2['bounds']), i1['payload']
+ i2['payload']))
print('is1:')
print(is1)
print('is2:')
print(is2)
print('is1 joined with is2:')
print(is3)
def test_len(self):
is1 = IntervalSet([
Interval(Bounds3D(1, 5)),
Interval(Bounds3D(10, 22)),
])
self.assertEqual(len(is1), 2)
self.assertEqual(is1.size(), 2)
def test_minus_self(self):
is1 = IntervalSet([
Interval(Bounds3D(2, 2.5)),
Interval(Bounds3D(2, 2.7)),
Interval(Bounds3D(2.9, 3.5)),
Interval(Bounds3D(3.5, 3.6)),
Interval(Bounds3D(5, 7)),
Interval(Bounds3D(9, 12)),
])
is1 = is1.minus(is1)
self.assertIntervalSetEq(is1, IntervalSet([]), eq)
def test_fold(self):
def fold_fn(acc, i):
return acc + (i['bounds'].length() * i['bounds'].width() *
i['bounds'].height())
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
self.assertAlmostEqual(is1.fold(fold_fn, 0), 1.375)
def test_filter(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
is1 = is1.filter(lambda i: i['payload'] > 2)
target = IntervalSet([
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
self.assertIntervalSetEq(is1, target, eq)
def test_join_with_optimization_window(self):
is1 = IntervalSet(
[Interval(Bounds3D(t, t + 1), t) for t in range(100)])
is2 = IntervalSet([Interval(Bounds3D(t, t), t) for t in range(100)])
is3 = is1.join(is2,
Bounds3D.T(before(max_dist=1)),
lambda i1, i2: Interval(i1['bounds'].span(i2['bounds']),
i2['payload']),
window=1)
target = IntervalSet(
[Interval(Bounds3D(t, t + 2), t + 2) for t in range(98)] +
[Interval(Bounds3D(t, t + 1), t + 1) for t in range(99)])
self.assertIntervalSetEq(is3, target, eq)
def test_map_payload(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1), payload=1),
Interval(Bounds3D(2, 3), payload=2),
Interval(Bounds3D(4, 5), payload=3),
])
is2 = is1.map_payload(lambda p: p + 10)
target = IntervalSet([
Interval(Bounds3D(0, 1), payload=11),
Interval(Bounds3D(2, 3), payload=12),
Interval(Bounds3D(4, 5), payload=13),
])
self.assertIntervalSetEq(is2, target)
def test_minus_everything(self):
is1 = IntervalSet(
[Interval(Bounds3D(1, 10)),
Interval(Bounds3D(3, 15))])
is2 = IntervalSet([
Interval(Bounds3D(2, 2.5)),
Interval(Bounds3D(2, 2.7)),
Interval(Bounds3D(2.9, 3.5)),
Interval(Bounds3D(3.5, 3.6)),
Interval(Bounds3D(5, 7)),
Interval(Bounds3D(9, 12)),
])
is3 = is2.minus(is1)
self.assertIntervalSetEq(is3, IntervalSet([]), eq)
def test_fold_custom_sortkey(self):
def sortkey(i):
return (i['x1'], i['x2'])
def fold_fn(acc, i):
acc.append(i['payload'])
return acc
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 0.1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0.1, 0.4, 0, 1, 0, 0.5), 3),
])
self.assertListEqual(is1.fold(fold_fn, [], sortkey), [1, 3, 2])
def test_minus_against_nothing(self):
is1 = IntervalSet([
Interval(Bounds3D(1, 10, 0, 0.5, 0.2, 0.8), 1),
Interval(Bounds3D(3, 15, 0, 1, 0, 1), 2)
])
is2 = IntervalSet([
Interval(Bounds3D(20, 20.5)),
Interval(Bounds3D(20, 20.7)),
Interval(Bounds3D(20.9, 23.5)),
Interval(Bounds3D(23.5, 23.6)),
Interval(Bounds3D(25, 27)),
Interval(Bounds3D(29, 32)),
])
is3 = is1.minus(is2)
self.assertIntervalSetEq(is3, is1, eq)
def test_filter_against(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1)),
Interval(Bounds3D(2, 5)),
Interval(Bounds3D(6, 7)),
])
is2 = IntervalSet([
Interval(Bounds3D(1, 1)),
Interval(Bounds3D(7, 7)),
])
# Take only intervals in is1 that overlaps with some interval in is2
is3 = is1.filter_against(is2, Bounds3D.T(overlaps()), window=0)
self.assertIntervalSetEq(
is3,
IntervalSet([Interval(Bounds3D(0, 1)),
Interval(Bounds3D(6, 7))]))
def query_that_throws_at_0(vids):
output = []
for vid in vids:
if vid == 0:
raise RuntimeError()
output.append(TestRuntime.dummy_interval(vid))
return IntervalSet(output)
def find_clips_for_keyword(keyword, use_only_video=False):
ism = {}
for vm in tqdm(video_metadata):
if not vm["annotation_filename"] or (not vm["only_video"]
and use_only_video):
continue
try:
h5 = eeghdf.Eeghdf(vm["annotation_filename"])
except:
print(vm["annotation_filename"])
os.remove(vm["annotation_filename"])
continue
starts = [start / 10**7 for start in h5._annotation_start100ns]
texts = h5._annotation_text
if not keyword or any(keyword.lower() in text.lower()
for text in texts):
interval_set = IntervalSet([
Interval(
Bounds3D(start, start + 5), # we set the duration
{
'spatial_type':
SpatialType_Caption(">>" + text + "\n"),
'metadata': {}
}) for start, text in zip(starts, texts)
])
ism[vm["id"]] = interval_set
print(
f"Found {len(ism)} videos with keyword {keyword} in the annotations.")
vgrid_spec = VGridSpec(video_meta=video_metadata_wrapper,
vis_format=VideoBlockFormat(imaps=[('bboxes',
ism)]),
video_endpoint='http://localhost:8080')
return VGridWidget(vgrid_spec=vgrid_spec.to_json_compressed())
def test_collect_by_interval(self):
c = TestIntervalSetMapping.get_collection()
d = IntervalSetMapping({1: IntervalSet([
Interval(Bounds3D(t,t)) for t in range(1, 100)])})
e = c.collect_by_interval(d, Bounds3D.T(overlaps()),
filter_empty=False, window=0)
self.assertEqual(e.keys(), c.keys())
def list_to_IntervalSetMapping(interval_list):
ism = {}
for video_id, start, end, duration in interval_list:
if not video_id in ism:
ism[video_id] = []
ism[video_id].append(Interval(Bounds3D(start, end)))
return IntervalSetMapping({video_id: IntervalSet(intervalSet) for video_id, intervalSet in ism.items()})
def get_maskrcnn_bboxes():
interval = 30
bboxes = [
pickle.loads(
requests.get(posixpath.join(
posixpath.join(VIDEO_COLLECTION_BASEURL, BBOX_FOLDER),
posixpath.join(vm.path[:-4], 'bboxes.pkl')),
verify=False).content) for vm in (video_metadata)
]
bboxes_ism = IntervalSetMapping({
metadata.id: IntervalSet([
Interval(Bounds3D(t1=30 * i / metadata.fps,
t2=30 * (i + interval) / metadata.fps,
x1=bbox[0] / metadata.width,
x2=bbox[2] / metadata.width,
y1=bbox[1] / metadata.height,
y2=bbox[3] / metadata.height),
payload={
'class': bbox[4],
'score': bbox[5],
'spatial_type': SpatialType_Bbox(text=bbox[4])
}) for i, frame in enumerate(bbox_frame_list)
if (i % interval == 0) for bbox in frame
])
for bbox_frame_list, metadata in tqdm(zip(bboxes, (video_metadata)),
total=len(bboxes))
})
return bboxes_ism
def get_ground_truth():
interval = 30
GT_FOLDER = 'empty_spaces'
empty_parking_spaces = [
pickle.loads(
requests.get(posixpath.join(
posixpath.join(VIDEO_COLLECTION_BASEURL, GT_FOLDER),
posixpath.join(vm.path[:-4], 'gt.pkl')),
verify=False).content) for vm in video_metadata
]
gt_ism = IntervalSetMapping({
metadata.id: IntervalSet([
Interval(
Bounds3D(t1=30 * i / metadata.fps,
t2=30 * (i + interval) / metadata.fps,
x1=bbox[0] / metadata.width + .01,
x2=bbox[2] / metadata.width - .01,
y1=bbox[1] / metadata.height + .01,
y2=bbox[3] / metadata.height - .01))
for i, frame in enumerate(space_frame_list) if (i % interval == 0)
for bbox in frame
])
for space_frame_list, metadata in tqdm(zip(empty_parking_spaces,
video_metadata),
total=len(empty_parking_spaces))
})
return gt_ism
def test_union(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 1),
])
is2 = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 2),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 2),
])
is3 = is1.union(is2)
target = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 2),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 1),
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 2),
])
self.assertIntervalSetEq(is3, target, eq)
def IntervalSetMapping_second_to_frame(ism):
intervalSets_frame = {}
for video_id, intervalSet in ism.get_grouped_intervals().items():
video = Video.objects.filter(id=video_id)[0]
fps = video.fps
intervalSets_frame[video_id] = IntervalSet(
[Interval(Bounds1D(int(i.bounds['t1'] * fps), int(i.bounds['t2'] * fps)), i.payload) \
for i in intervalSet.get_intervals()] )
return IntervalSetMapping(intervalSets_frame)
def split_fn(i):
output = []
t = i.copy()
while t['bounds'].length() > 5:
output.append(Interval(Bounds3D(t['t1'], t['t1'] + 5)))
t['t1'] = t['t1'] + 5
if t['bounds'].length() > 0:
output.append(t)
return IntervalSet(output)
def interval_blocks(self):
return [
IntervalBlock(
video_id=video_key,
interval_sets=[
NamedIntervalSet(name='default', interval_set=IntervalSet([interval]))
])
for video_key in self._imap
for interval in self._imap[video_key].get_intervals()
] # yapf: disable
def rs_to_rekall(rs_ilist, video_ids=None, with_payload=True):
rekall_ism = {}
if video_ids is None:
video_ids = rs_ilist.get_ids()
for video_id in video_ids:
if with_payload:
interval_list = rs_ilist.get_intervals_with_payload(video_id, True)
rekall_ism[video_id] = IntervalSet([
Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0),
i[2]) for i in interval_list
])
else:
interval_list = rs_ilist.get_intervals(video_id, True)
rekall_ism[video_id] = IntervalSet([
Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0))
for i in interval_list
])
return IntervalSetMapping(rekall_ism)
def test_join(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
])
is2 = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 4),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 8),
])
is3 = is1.join(
is2, Bounds3D.T(overlaps()), lambda i1, i2: Interval(
i1['bounds'].intersect_time_span_space(i2['bounds']), i1[
'payload'] + i2['payload']))
target = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 5),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 9),
Interval(Bounds3D(0, 0.5, 0, 1, 0, 1), 10),
])
self.assertIntervalSetEq(is3, target, eq)
def test_map(self):
def expand_to_frame(intrvl):
new_bounds = intrvl['bounds'].copy()
new_bounds['x1'] = 0
new_bounds['x2'] = 1
new_bounds['y1'] = 0
new_bounds['y2'] = 1
return Interval(new_bounds, intrvl['payload'])
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0.3, 0.4, 0.5, 0.6)),
Interval(Bounds3D(0, 0.5, 0.2, 0.3, 0.5, 0.6))
])
is1 = is1.map(expand_to_frame)
target = IntervalSet([
Interval(Bounds3D(0, 0.5, 0, 1, 0, 1)),
Interval(Bounds3D(0, 1, 0, 1, 0, 1)),
])
self.assertIntervalSetEq(is1, target)
def for_vgrid(self):
fps = self.obj.video.fps
pad = 0.02
return IntervalSet([
Interval(
Bounds3D(t1=p.frame / fps,
t2=(p.frame + 1) / fps,
x1=p.to_pixel_norm()[0] - pad,
x2=p.to_pixel_norm()[0] + pad,
y1=p.to_pixel_norm()[1] - pad,
y2=p.to_pixel_norm()[1] + pad)) for p in self.pos
])
def test_group_by(self):
is1 = IntervalSet([
Interval(Bounds3D(0, 1)),
Interval(Bounds3D(1, 2)),
Interval(Bounds3D(0, 2)),
Interval(Bounds3D(1, 3)),
])
def merge_intervals(k, intervals):
merged = intervals.fold(
lambda i1, i2: Interval(i1['bounds'].span(i2['bounds'])))
merged['payload'] = intervals
return merged
is2 = is1.group_by(lambda i: int(i['t2']) % 2, merge_intervals)
target = IntervalSet([
Interval(Bounds3D(0, 3),
payload=IntervalSet([
Interval(Bounds3D(0, 1)),
Interval(Bounds3D(1, 3)),
])),
Interval(Bounds3D(0, 2),
payload=IntervalSet([
Interval(Bounds3D(0, 2)),
Interval(Bounds3D(1, 2)),
])),
])
self.assertIntervalSetEq(is2, target,
self.compare_interval_sets_in_payload())
def __init__(self, block_label_state):
self._new_intervals = IntervalSet([
Interval(
Bounds3D(t1=intvl['bounds']['t1'],
t2=intvl['bounds']['t2'],
x1=intvl['bounds']['bbox']['x1'],
x2=intvl['bounds']['bbox']['x2'],
y1=intvl['bounds']['bbox']['y1'],
y2=intvl['bounds']['bbox']['y2']))
for intvl in block_label_state['new_intervals']
])
self._captions_selected = None # TODO
def test_match(self):
left_box_frame_1 = Interval(Bounds3D(1, 1, 0.1, 0.4, 0.4, 0.8))
right_box_frame_1 = Interval(Bounds3D(1, 1, 0.6, 0.9, 0.3, 0.7))
bottom_left_box_frame_2 = Interval(Bounds3D(2, 2, 0.1, 0.3, 0.8, 0.9))
top_right_box_frame_2 = Interval(Bounds3D(2, 2, 0.5, 0.7, 0.2, 0.7))
is1 = IntervalSet([
left_box_frame_1, right_box_frame_1, bottom_left_box_frame_2,
top_right_box_frame_2
])
pattern = [
(
["left", "right"],
[
# Two boxes on left and right on same frame
Bounds3D.T(equal()),
Bounds3D.XY(left_of()),
]),
(
["top", "bottom"],
[
# Two boxes on top and bottom on overlapping frame
Bounds3D.T(equal()),
Bounds3D.XY(above()),
]),
(
["left", "top"],
[
# Left-Right pattern comes before Top-Bottom
Bounds3D.T(meets_before(epsilon=1))
])
]
results = is1.match(pattern, exact=True)
self.assertEqual(len(results), 1)
result = results[0]
self.assertIntervalsEq(left_box_frame_1, result['left'])
self.assertIntervalsEq(right_box_frame_1, result['right'])
self.assertIntervalsEq(top_right_box_frame_2, result['top'])
self.assertIntervalsEq(bottom_left_box_frame_2, result['bottom'])