def process_single(black_frames_path, captions_path, metadata_path,
commercials_outpath):
# Load original data
black_frames = load_json(black_frames_path)
captions = load_captions(captions_path)
metadata = load_json(metadata_path)
# Create IntervalSet objects
black_frames_set = IntervalSet([
Interval(
Bounds3D(frame_num / metadata['fps'],
(frame_num + 1) / metadata['fps']))
for frame_num in black_frames
])
captions_set = IntervalSet([
Interval(Bounds3D(start, end), payload=text)
for text, start, end, in captions
])
whole_video = IntervalSet(
[Interval(Bounds3D(0, metadata['frames'] / metadata['fps']))])
# Detect commercials
results = detect_commercials(black_frames_set, captions_set, whole_video)
# Convert commercial intervals back to frames
results = convert_set_from_seconds_to_frames(results, metadata['fps'])
# Save results in JSON format
results = [(r['t1'], r['t2']) for r in results.get_intervals()]
save_json(results, commercials_outpath)
def fold_fn(stack, interval):
if interval['t2'] - interval['t1'] > MAX_COMMERCIAL_TIME:
interval = Interval(
Bounds3D(interval['t1'], interval['t1'] + MAX_COMMERCIAL_TIME))
if len(stack) == 0:
stack.append(interval)
else:
last = stack.pop()
if or_pred(overlaps(),
after(max_dist=COMMERCIAL_FOLD_EPSILON))(interval,
last):
if last['bounds'].span(interval['bounds']).size() \
> MAX_COMMERCIAL_TIME:
stack.append(
Interval(
Bounds3D(last['t1'],
last['t1'] + MAX_COMMERCIAL_TIME)))
else:
stack.append(
Interval(last['bounds'].span(interval['bounds'])))
else:
stack.append(last)
stack.append(interval)
return stack
def test_coalesce_iou_distance(self):
input_list = self.intrvl_list_iou_distance
target = [
Interval(Bounds3D(0, 11, 0., .52, 0., .52),
{'msg': 'first person starts at frame 0'}),
Interval(
Bounds3D(5, 6, .1, .6, .1, .6), {
'msg':
'second person starts at frame 5 and overlaps with first too'
}),
]
output = Coalesce(
axis=('t1', 't2'),
bounds_merge_op=Bounds3D.span,
predicate=and_pred(
before(),
iou_at_least(0.001)), # both persons at frame 5 should pass
epsilon=6,
distance=_iou)(FromIterable(input_list)())
results = run_to_finish(output)
self.assertIntervalListEq(results, target, bounds_only=True)
def main():
'''
An example of a join. We filter pairs based on time overlap, and merge
surviving pairs by taking the intersection over the time bounds and the
span of the spatial bounds (and adding the payloads).
'''
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
])
is2 = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 4),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 8),
])
is3 = is1.join(
is2, Bounds3D.T(overlaps()), lambda i1, i2: Interval(
i1['bounds'].intersect_time_span_space(i2['bounds']), i1['payload']
+ i2['payload']))
print('is1:')
print(is1)
print('is2:')
print(is2)
print('is1 joined with is2:')
print(is3)
def test_set_coalesce_some_overlap(self):
output = SetCoalesce(axis=('t1', 't2'), bounds_merge_op=Bounds3D.span)(
FromIterable(self.intrvl_list_some_overlap)())
results = run_to_finish(output)
L = self.intrvl_list_some_overlap
self.assertIntervalListEq(
results,
[
Interval(Bounds3D(0, 50), L[0].payload), # 0 and 1 merge
Interval(Bounds3D(53, 60), L[2].payload), # 2 unchanged
Interval(Bounds3D(1000, 2999), L[3].payload), # 3 and 4 merge
])
def test_small_window(self):
# the given window is smaller than what the input satisfies
# thus it won't wait for t1=0 before releasing t1=20
output = BoundedSort(window=10)(FromIterable(self.intrvl_list_2)())
results = run_to_finish(output)
expected = [
Interval(Bounds3D(20, 30)), # pre-mature release
Interval(Bounds3D(0, 10)),
Interval(Bounds3D(40, 50)),
]
self.assertIntervalListEq(results, expected)
def test_overlaptime_antijoin(self):
input_left = FromIterable(self.intrvl_list_1)()
input_right = FromIterable(self.intrvl_list_2)()
target = [
# alpha should be dropped because it matches first
Interval(Bounds3D(20, 30), {'msg': 'beta'}),
Interval(Bounds3D(40, 50), {'msg': 'gamma'}),
]
output = AntiJoinWithTimeWindow(predicate=overlaps(), )(input_left,
input_right)
results = run_to_finish(output)
self.assertIntervalListEq(results, target)
def find_clips_for_keyword(keyword, use_only_video=False):
ism = {}
for vm in tqdm(video_metadata):
if not vm["annotation_filename"] or (not vm["only_video"]
and use_only_video):
continue
try:
h5 = eeghdf.Eeghdf(vm["annotation_filename"])
except:
print(vm["annotation_filename"])
os.remove(vm["annotation_filename"])
continue
starts = [start / 10**7 for start in h5._annotation_start100ns]
texts = h5._annotation_text
if not keyword or any(keyword.lower() in text.lower()
for text in texts):
interval_set = IntervalSet([
Interval(
Bounds3D(start, start + 5), # we set the duration
{
'spatial_type':
SpatialType_Caption(">>" + text + "\n"),
'metadata': {}
}) for start, text in zip(starts, texts)
])
ism[vm["id"]] = interval_set
print(
f"Found {len(ism)} videos with keyword {keyword} in the annotations.")
vgrid_spec = VGridSpec(video_meta=video_metadata_wrapper,
vis_format=VideoBlockFormat(imaps=[('bboxes',
ism)]),
video_endpoint='http://localhost:8080')
return VGridWidget(vgrid_spec=vgrid_spec.to_json_compressed())
def get_ground_truth():
interval = 30
GT_FOLDER = 'empty_spaces'
empty_parking_spaces = [
pickle.loads(
requests.get(posixpath.join(
posixpath.join(VIDEO_COLLECTION_BASEURL, GT_FOLDER),
posixpath.join(vm.path[:-4], 'gt.pkl')),
verify=False).content) for vm in video_metadata
]
gt_ism = IntervalSetMapping({
metadata.id: IntervalSet([
Interval(
Bounds3D(t1=30 * i / metadata.fps,
t2=30 * (i + interval) / metadata.fps,
x1=bbox[0] / metadata.width + .01,
x2=bbox[2] / metadata.width - .01,
y1=bbox[1] / metadata.height + .01,
y2=bbox[3] / metadata.height - .01))
for i, frame in enumerate(space_frame_list) if (i % interval == 0)
for bbox in frame
])
for space_frame_list, metadata in tqdm(zip(empty_parking_spaces,
video_metadata),
total=len(empty_parking_spaces))
})
return gt_ism
def test_overlaptime_join(self):
input_left = FromIterable(self.intrvl_list_1)()
input_right = FromIterable(self.intrvl_list_2)()
target = [
Interval(Bounds3D(0, 12), {'msg': 'alpha|first'}),
Interval(Bounds3D(0, 25), {'msg': 'alpha|second'}),
Interval(Bounds3D(8, 30), {'msg': 'beta|second'}),
Interval(Bounds3D(40, 50), {'msg': 'gamma|third'}),
]
output = JoinWithTimeWindow(predicate=overlaps(),
merge_op=TestJoinWithTimeWindow.merge_msg)(
input_left, input_right)
results = run_to_finish(output)
self.assertIntervalListEq(results, target)
def get_maskrcnn_bboxes():
interval = 30
bboxes = [
pickle.loads(
requests.get(posixpath.join(
posixpath.join(VIDEO_COLLECTION_BASEURL, BBOX_FOLDER),
posixpath.join(vm.path[:-4], 'bboxes.pkl')),
verify=False).content) for vm in (video_metadata)
]
bboxes_ism = IntervalSetMapping({
metadata.id: IntervalSet([
Interval(Bounds3D(t1=30 * i / metadata.fps,
t2=30 * (i + interval) / metadata.fps,
x1=bbox[0] / metadata.width,
x2=bbox[2] / metadata.width,
y1=bbox[1] / metadata.height,
y2=bbox[3] / metadata.height),
payload={
'class': bbox[4],
'score': bbox[5],
'spatial_type': SpatialType_Bbox(text=bbox[4])
}) for i, frame in enumerate(bbox_frame_list)
if (i % interval == 0) for bbox in frame
])
for bbox_frame_list, metadata in tqdm(zip(bboxes, (video_metadata)),
total=len(bboxes))
})
return bboxes_ism
def test_fold_number_min(self):
input_list = self.intrvl_list_number
target = [
Interval(Bounds3D(0, 0), {'number': 2}),
]
output = Fold(
update_fn=lambda state, intrvl: min(state, intrvl.payload['number']
),
init_state=float('inf'),
finalize_fn=lambda state: Interval(Bounds3D(0, 0),
{'number': state}),
)(FromIterable(input_list)())
results = run_to_finish(output)
self.assertIntervalListEq(results, target)
def test_cartessian_join_with_small_window(self):
input_left = FromIterable(self.intrvl_list_1)()
input_right = FromIterable(self.intrvl_list_2)()
window = 30
target = [
# alpha should not join third and fourth
Interval(Bounds3D(0, 12), {'msg': 'alpha|first'}),
Interval(Bounds3D(0, 25), {'msg': 'alpha|second'}),
# beta should not join fourth
Interval(Bounds3D(7, 30), {'msg': 'beta|first'}),
Interval(Bounds3D(8, 30), {'msg': 'beta|second'}),
Interval(Bounds3D(20, 50), {'msg': 'beta|third'}),
# gamma should not join first and second
Interval(Bounds3D(40, 50), {'msg': 'gamma|third'}),
Interval(Bounds3D(40, 70), {'msg': 'gamma|fourth'}),
]
sorted_target = sorted(target, key=lambda i: (i['t1'], i['t2']))
output = JoinWithTimeWindow(predicate=lambda i1, i2: True,
merge_op=TestJoinWithTimeWindow.merge_msg,
window=window)(input_left, input_right)
results = run_to_finish(output)
self.assertIntervalListEq(results, sorted_target)
def test_fold_number_avg(self):
input_list = self.intrvl_list_number
target = [
Interval(Bounds3D(0, 0), {'number': 33}),
]
# state = (count, sum)
output = Fold(
update_fn=lambda state, intrvl:
(state[0] + 1, state[1] + intrvl.payload['number']),
init_state=(0, 0.),
finalize_fn=lambda state: Interval(Bounds3D(
0, 0), {'number': state[1] / state[0]}),
)(FromIterable(input_list)())
results = run_to_finish(output)
self.assertIntervalListEq(results, target)
def rs_to_rekall(rs_ilist, video_ids=None, with_payload=True):
rekall_ism = {}
if video_ids is None:
video_ids = rs_ilist.get_ids()
for video_id in video_ids:
if with_payload:
interval_list = rs_ilist.get_intervals_with_payload(video_id, True)
rekall_ism[video_id] = IntervalSet([
Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0),
i[2]) for i in interval_list
])
else:
interval_list = rs_ilist.get_intervals(video_id, True)
rekall_ism[video_id] = IntervalSet([
Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0))
for i in interval_list
])
return IntervalSetMapping(rekall_ism)
def test_coalesce_iou_early_late_epsilon(self):
input_list = self.intrvl_list_iou_early_late_epsilon
target = [
Interval(Bounds3D(0, 11, 0., .52, 0., .52),
{'msg': 'first person .'}),
Interval(Bounds3D(3, 8, 0.49, 1., .49, 1.),
{'msg': 'second person .'}),
Interval(Bounds3D(100, 120, 0., .8, .0, .8),
{'msg': 'third person starts .'}),
]
output = Coalesce(axis=('t1', 't2'),
bounds_merge_op=Bounds3D.span,
predicate=iou_at_least(0.8),
epsilon=6)(FromIterable(input_list)())
results = run_to_finish(output)
# print(results)
self.assertIntervalListEq(results, target, bounds_only=True)
def for_vgrid(self):
fps = self.obj.video.fps
pad = 0.02
return IntervalSet([
Interval(
Bounds3D(t1=p.frame / fps,
t2=(p.frame + 1) / fps,
x1=p.to_pixel_norm()[0] - pad,
x2=p.to_pixel_norm()[0] + pad,
y1=p.to_pixel_norm()[1] - pad,
y2=p.to_pixel_norm()[1] + pad)) for p in self.pos
])
def test_set_coalesce_all_overlap_payload_merge(self):
def payload_merge_msg(p1, p2):
return {'msg': p1['msg'] + '|' + p2['msg']}
output = SetCoalesce(axis=('t1', 't2'),
bounds_merge_op=Bounds3D.span,
payload_merge_op=payload_merge_msg)(FromIterable(
self.intrvl_list_all_overlap)())
results = run_to_finish(output)
self.assertIntervalListEq(
results,
[Interval(Bounds3D(0, 60), {'msg': 'hello|world|hello world'})])
def test_set_coalesce_some_overlap_epsilon_2(self):
output = SetCoalesce(axis=('t1', 't2'),
bounds_merge_op=Bounds3D.span,
epsilon=999)(FromIterable(
self.intrvl_list_some_overlap)())
results = run_to_finish(output)
L = self.intrvl_list_some_overlap
self.assertIntervalListEq(
results,
[
Interval(Bounds3D(0, 2999), L[0].payload), # all should merge
])
def __init__(self, block_label_state):
self._new_intervals = IntervalSet([
Interval(
Bounds3D(t1=intvl['bounds']['t1'],
t2=intvl['bounds']['t2'],
x1=intvl['bounds']['bbox']['x1'],
x2=intvl['bounds']['bbox']['x2'],
y1=intvl['bounds']['bbox']['y1'],
y2=intvl['bounds']['bbox']['y2']))
for intvl in block_label_state['new_intervals']
])
self._captions_selected = None # TODO
def test_equitime_join(self):
input_left = FromIterable(self.intrvl_list_1)()
input_right = FromIterable(self.intrvl_list_2)()
target = [
Interval(Bounds3D(40, 50), {'msg': 'gamma|third'}),
]
output = JoinWithTimeWindow(predicate=equal(),
merge_op=TestJoinWithTimeWindow.merge_msg)(
input_left, input_right)
results = run_to_finish(output)
self.assertIntervalListEq(results, target)
def main():
'''
Examples of simple functions on IntervalSets.
'''
# filter examples
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3),
])
print('is1:')
print(is1)
is1_filtered = is1.filter(lambda i: i['payload'] > 2)
print('is1 filtered on payload:')
print(is1_filtered)
print()
# map example
def expand_to_frame(intrvl):
new_bounds = intrvl['bounds'].copy()
new_bounds['x1'] = 0
new_bounds['x2'] = 1
new_bounds['y1'] = 0
new_bounds['y2'] = 1
return Interval(new_bounds, intrvl['payload'])
is2 = IntervalSet([
Interval(Bounds3D(0, 1, 0.3, 0.4, 0.5, 0.6)),
Interval(Bounds3D(0, 0.5, 0.2, 0.3, 0.5, 0.6))
])
print('is2:')
print(is2)
is2_expanded = is2.map(expand_to_frame)
print('is2 expanded to the frame:')
print(is2_expanded)
def main():
'''
Examples of creating some Intervals. Intervals are parameterized by Bounds
(Bounds3D in these examples) and an optional payload.
'''
my_first_bounds = Bounds3D(t1=5, t2=10, x1=0.1, x2=0.4, y1=0.3, y2=0.9)
# This bounds is the same as my_first_bounds
my_second_bounds = Bounds3D(5, 10, 0.1, 0.4, 0.3, 0.9)
# This Interval has the 3D bounds of my_first_bounds and a payload
my_first_interval = Interval(my_first_bounds, payload=10)
# This Interval doesn't have a payload
my_first_interval_no_payload = Interval(my_first_bounds)
# IntervalSet contains multiple Intervals
my_first_interval_set = IntervalSet([
my_first_interval,
Interval(my_second_bounds, payload='payloads are arbitrary'),
Interval(Bounds3D(20, 30), payload='Bounds3D has default X/Y fields')
])
print(my_first_interval_set)
class TestAntiJoinWithTimeWindow(OpTestCase):
intrvl_list_1 = [
Interval(Bounds3D(0, 10), {'msg': 'alpha'}),
Interval(Bounds3D(20, 30), {'msg': 'beta'}),
Interval(Bounds3D(40, 50), {'msg': 'gamma'}),
]
intrvl_list_2 = [
Interval(Bounds3D(7, 12), {'msg': 'first'}),
Interval(Bounds3D(60, 70), {'msg': 'fourth'}),
]
intrvl_list_3 = [
Interval(Bounds3D(7, 45), {'msg': 'bang!'}),
]
def test_overlaptime_antijoin(self):
input_left = FromIterable(self.intrvl_list_1)()
input_right = FromIterable(self.intrvl_list_2)()
target = [
# alpha should be dropped because it matches first
Interval(Bounds3D(20, 30), {'msg': 'beta'}),
Interval(Bounds3D(40, 50), {'msg': 'gamma'}),
]
output = AntiJoinWithTimeWindow(predicate=overlaps(), )(input_left,
input_right)
results = run_to_finish(output)
self.assertIntervalListEq(results, target)
def test_pass_all_antijoin(self):
input_left = FromIterable(self.intrvl_list_1)()
input_right = FromIterable(self.intrvl_list_2)()
target = self.intrvl_list_1
output = AntiJoinWithTimeWindow(
predicate=lambda i1, i2: False, # don't drop any
)(input_left, input_right)
results = run_to_finish(output)
self.assertIntervalListEq(results, target)
def test_drop_all_antijoin(self):
input_left = FromIterable(self.intrvl_list_1)()
input_right = FromIterable(self.intrvl_list_3)()
target = []
output = AntiJoinWithTimeWindow(predicate=overlaps())(input_left,
input_right)
results = run_to_finish(output)
self.assertIntervalListEq(results, target)
def all_videos():
from scripts.models import Video
from vgrid import VideoVBlocksBuilder, VideoTrackBuilder, VideoMetadata
from rekall import IntervalSet, Interval, Bounds3D, IntervalSetMapping
videos = Video.objects.all()
json = VideoVBlocksBuilder() \
.add_track(VideoTrackBuilder(
'videos', IntervalSetMapping({v.id: IntervalSet([Interval(Bounds3D(0, v.duration()))])
for v in videos}))) \
.add_video_metadata(
'/system_media', [VideoMetadata(**{k: getattr(v, k)
for k in ['id', 'path', 'num_frames', 'fps', 'width', 'height']})
for v in videos]) \
.build()
return json
def get_maskrcnn_bboxes():
maskrcnn_bboxes_ism = IntervalSetMapping({
vm.id: IntervalSet([
Interval(Bounds3D(t1=frame_num / vm.fps,
t2=(frame_num + 1) / vm.fps,
x1=bbox[0] / vm.width,
x2=bbox[2] / vm.width,
y1=bbox[1] / vm.height,
y2=bbox[3] / vm.height),
payload={
'class': bbox[4],
'score': bbox[5]
})
for frame_num, bboxes_in_frame in enumerate(maskrcnn_frame_list)
for bbox in bboxes_in_frame
])
for vm, maskrcnn_frame_list in zip(video_metadata_intel,
maskrcnn_bboxes_intel)
})
return maskrcnn_bboxes_ism
][::stride]
video_meta = video_path_to_vm(
json_path_to_video_path(pose_annotation_file)
)
if video_meta is None:
continue
for frame_number, pose_annotation in tqdm(enumerate(pose_annotations), total=len(pose_annotations)):
start = (frame_number * stride) / video_meta.fps
end = (frame_number + 1) * stride / video_meta.fps
for pose in pose_annotation:
pose_intervals.append(
Interval(
Bounds3D(start, end),
{
'spatial_type': SpatialType_Keypoints(),
'metadata': {
'pose': Metadata_Keypoints.from_openpose(
pose_annotation_to_array(pose)
)
}
}
)
)
pose_metadata[video_meta.id] = IntervalSet(pose_intervals)
# In[29]:
from rekall import Interval, IntervalSet, IntervalSetMapping, Bounds3D
# Vgrid imports
from vgrid import VGridSpec, VideoMetadata, VideoBlockFormat
from vgrid import SpatialType_Temporal, Metadata_Generic
# This example assumes a 1920x1080 video at 59.94 FPS, with 20,696 frames.
# You should modify it for your own examples.
video_metadata = [
VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920, 1080)
]
ism = IntervalSetMapping({
0: IntervalSet([
Interval(
Bounds3D(0, 10),
{
'spatial_type': SpatialType_Temporal(),
'metadata': {
'generic_metadata': Metadata_Generic(
'this will be drawn below the timeline when expanded'
),
'other_generic_metadata': Metadata_Generic({
'key': 'this object will also be stringified'
})
}
}
)
])
})
'''
This example shows how to draw a simple bounding box with text when expanded.
'''
# Rekall imports
from rekall import Interval, IntervalSet, IntervalSetMapping, Bounds3D
# Vgrid imports
from vgrid import VGridSpec, VideoMetadata, VideoBlockFormat, SpatialType_Bbox
# This example assumes a 1920x1080 video at 59.94 FPS, with 20,696 frames.
# You should modify it for your own examples.
video_metadata = [
VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920,
1080)
]
ism = IntervalSetMapping({
0:
IntervalSet([
Interval(Bounds3D(0, 10),
{'spatial_type': SpatialType_Bbox(text='my text')})
])
})
vgrid_spec = VGridSpec(video_meta=video_metadata,
vis_format=VideoBlockFormat(imaps=[('bboxes', ism)]))
# Pass this to your Javascript application somehow
json_for_js = vgrid_spec.to_json_compressed()
def main():
'''
Examples of minus (anti-semi-join).
The first example subtracts out all the parts of the Intervals in the left
set that overlap with any part of an interval in the right set.
The second example uses a predicate to only subtract Intervals with the
same payload.
'''
is1 = IntervalSet([
Interval(Bounds3D(1, 10,0,0.5,0.2,0.8),1),
Interval(Bounds3D(3, 15,0,1,0,1),2)
])
is2 = IntervalSet([
Interval(Bounds3D(2,2.5),1),
Interval(Bounds3D(2,2.7),1),
Interval(Bounds3D(2.9,3.5),1),
Interval(Bounds3D(3.5,3.6),1),
Interval(Bounds3D(5,7),2),
Interval(Bounds3D(9,12),2),
])
is3 = is1.minus(is2)
# is3 is equal to target
target = IntervalSet([
Interval(Bounds3D(1,2,0,0.5,0.2,0.8),1),
Interval(Bounds3D(2.7, 2.9, 0,0.5,0.2,0.8),1),
Interval(Bounds3D(3.6,5,0,0.5,0.2,0.8),1),
Interval(Bounds3D(7,9,0,0.5,0.2,0.8),1),
Interval(Bounds3D(3.6,5), payload=2),
Interval(Bounds3D(7,9), payload=2),
Interval(Bounds3D(12,15), payload=2),
])
is4 = is1.minus(is2, predicate=payload_satisfies(
lambda p1, p2: p1 == p2
))
# is4 is equal to target2
target2 = IntervalSet([
Interval(Bounds3D(1,2,0,0.5,0.2,0.8),1),
Interval(Bounds3D(2.7, 2.9, 0,0.5,0.2,0.8),1),
Interval(Bounds3D(3.6,10,0,0.5,0.2,0.8),1),
Interval(Bounds3D(3,5), payload=2),
Interval(Bounds3D(7,9), payload=2),
Interval(Bounds3D(12,15), payload=2),
])
print('is1:')
print(is1)
print('is2:')
print(is2)
print()
print('is1 minus is2:')
print(is3)
print()
print('is1 minus is2 filtering on the payload:')
print(is4)