def test_minus(self):
c= TestIntervalSetMapping.get_collection()
c1 = IntervalSetMapping({v: c[v] for v in c if v % 2 ==0})
c2 = IntervalSetMapping({v: c[v] for v in c if v % 2 ==1})
c3 = c.minus(c1, window=0)
self.assertCollectionEq(c3,c2)
def disjoint_domain_combiner(result1, result2):
""" A faster combiner than union_combiner for IntervalSetMapping.
Assumes that the results are of type IntervalSetMapping and every
chunk produces its own set of domain keys that are disjoint from results of
other chunks.
Args:
result1, result2 (IntervalSetMapping): partial results from
some chunks of total work.
Returns:
A IntervalSetMapping that is the union of the two.
Raises:
RekallRuntimeException: Raised if results have common domain key.
"""
d1 = result1.get_grouped_intervals()
d2 = result2.get_grouped_intervals()
k1, k2 = set(d1.keys()), set(d2.keys())
if k1.isdisjoint(k2):
return IntervalSetMapping({**d1, **d2})
intersection = k1 & k2
raise RekallRuntimeException(
"DisjointDomainCombiner used on results"
" with overlapping domains {0}".format(intersection))
def get_ground_truth():
interval = 30
GT_FOLDER = 'empty_spaces'
empty_parking_spaces = [
pickle.loads(
requests.get(posixpath.join(
posixpath.join(VIDEO_COLLECTION_BASEURL, GT_FOLDER),
posixpath.join(vm.path[:-4], 'gt.pkl')),
verify=False).content) for vm in video_metadata
]
gt_ism = IntervalSetMapping({
metadata.id: IntervalSet([
Interval(
Bounds3D(t1=30 * i / metadata.fps,
t2=30 * (i + interval) / metadata.fps,
x1=bbox[0] / metadata.width + .01,
x2=bbox[2] / metadata.width - .01,
y1=bbox[1] / metadata.height + .01,
y2=bbox[3] / metadata.height - .01))
for i, frame in enumerate(space_frame_list) if (i % interval == 0)
for bbox in frame
])
for space_frame_list, metadata in tqdm(zip(empty_parking_spaces,
video_metadata),
total=len(empty_parking_spaces))
})
return gt_ism
def get_maskrcnn_bboxes():
interval = 30
bboxes = [
pickle.loads(
requests.get(posixpath.join(
posixpath.join(VIDEO_COLLECTION_BASEURL, BBOX_FOLDER),
posixpath.join(vm.path[:-4], 'bboxes.pkl')),
verify=False).content) for vm in (video_metadata)
]
bboxes_ism = IntervalSetMapping({
metadata.id: IntervalSet([
Interval(Bounds3D(t1=30 * i / metadata.fps,
t2=30 * (i + interval) / metadata.fps,
x1=bbox[0] / metadata.width,
x2=bbox[2] / metadata.width,
y1=bbox[1] / metadata.height,
y2=bbox[3] / metadata.height),
payload={
'class': bbox[4],
'score': bbox[5],
'spatial_type': SpatialType_Bbox(text=bbox[4])
}) for i, frame in enumerate(bbox_frame_list)
if (i % interval == 0) for bbox in frame
])
for bbox_frame_list, metadata in tqdm(zip(bboxes, (video_metadata)),
total=len(bboxes))
})
return bboxes_ism
def test_collect_by_interval(self):
c = TestIntervalSetMapping.get_collection()
d = IntervalSetMapping({1: IntervalSet([
Interval(Bounds3D(t,t)) for t in range(1, 100)])})
e = c.collect_by_interval(d, Bounds3D.T(overlaps()),
filter_empty=False, window=0)
self.assertEqual(e.keys(), c.keys())
def list_to_IntervalSetMapping(interval_list):
ism = {}
for video_id, start, end, duration in interval_list:
if not video_id in ism:
ism[video_id] = []
ism[video_id].append(Interval(Bounds3D(start, end)))
return IntervalSetMapping({video_id: IntervalSet(intervalSet) for video_id, intervalSet in ism.items()})
def IntervalSetMapping_second_to_frame(ism):
intervalSets_frame = {}
for video_id, intervalSet in ism.get_grouped_intervals().items():
video = Video.objects.filter(id=video_id)[0]
fps = video.fps
intervalSets_frame[video_id] = IntervalSet(
[Interval(Bounds1D(int(i.bounds['t1'] * fps), int(i.bounds['t2'] * fps)), i.payload) \
for i in intervalSet.get_intervals()] )
return IntervalSetMapping(intervalSets_frame)
def get_collection():
NUM_INTERVALS=150
NUM_VIDEOS=100
def get_intervals_for_video(vid):
intervals = [
Interval(Bounds3D(t, t+1,0,0.5,0.5,1),vid)
for t in range(vid, vid+NUM_INTERVALS)]
return IntervalSet(intervals)
return IntervalSetMapping(
{vid: get_intervals_for_video(vid) for vid in range(
NUM_VIDEOS)})
def all_videos():
from scripts.models import Video
from vgrid import VideoVBlocksBuilder, VideoTrackBuilder, VideoMetadata
from rekall import IntervalSet, Interval, Bounds3D, IntervalSetMapping
videos = Video.objects.all()
json = VideoVBlocksBuilder() \
.add_track(VideoTrackBuilder(
'videos', IntervalSetMapping({v.id: IntervalSet([Interval(Bounds3D(0, v.duration()))])
for v in videos}))) \
.add_video_metadata(
'/system_media', [VideoMetadata(**{k: getattr(v, k)
for k in ['id', 'path', 'num_frames', 'fps', 'width', 'height']})
for v in videos]) \
.build()
return json
def rs_to_rekall(rs_ilist, video_ids=None, with_payload=True):
rekall_ism = {}
if video_ids is None:
video_ids = rs_ilist.get_ids()
for video_id in video_ids:
if with_payload:
interval_list = rs_ilist.get_intervals_with_payload(video_id, True)
rekall_ism[video_id] = IntervalSet([
Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0),
i[2]) for i in interval_list
])
else:
interval_list = rs_ilist.get_intervals(video_id, True)
rekall_ism[video_id] = IntervalSet([
Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0))
for i in interval_list
])
return IntervalSetMapping(rekall_ism)
def get_maskrcnn_bboxes():
maskrcnn_bboxes_ism = IntervalSetMapping({
vm.id: IntervalSet([
Interval(Bounds3D(t1=frame_num / vm.fps,
t2=(frame_num + 1) / vm.fps,
x1=bbox[0] / vm.width,
x2=bbox[2] / vm.width,
y1=bbox[1] / vm.height,
y2=bbox[3] / vm.height),
payload={
'class': bbox[4],
'score': bbox[5]
})
for frame_num, bboxes_in_frame in enumerate(maskrcnn_frame_list)
for bbox in bboxes_in_frame
])
for vm, maskrcnn_frame_list in zip(video_metadata_intel,
maskrcnn_bboxes_intel)
})
return maskrcnn_bboxes_ism
# Rekall imports
from rekall import Interval, IntervalSet, IntervalSetMapping, Bounds3D
# Vgrid imports
from vgrid import VGridSpec, VideoMetadata, VideoBlockFormat
from vgrid import SpatialType_Temporal
# This example assumes a 1920x1080 video at 59.94 FPS, with 20,696 frames.
# You should modify it for your own examples.
video_metadata = [
VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920,
1080)
]
ism = IntervalSetMapping({
0:
IntervalSet([
Interval(Bounds3D(0, 10), {
'spatial_type': SpatialType_Temporal(),
'metadata': {}
})
])
})
vgrid_spec = VGridSpec(video_meta=video_metadata,
vis_format=VideoBlockFormat(imaps=[('bboxes', ism)]))
# Pass this to your Javascript application somehow
json_for_js = vgrid_spec.to_json_compressed()
def query(vids):
return IntervalSetMapping(
{vid: TestRuntime.dummy_intervalset()
for vid in vids})
VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920,
1080)
]
ism = IntervalSetMapping({
0:
IntervalSet([
Interval(
Bounds3D(0, 10), {
'spatial_type': SpatialType_Caption('this is caption text'),
'metadata': {}
}),
Interval(
Bounds3D(10, 20), {
'spatial_type':
SpatialType_Caption('this is more caption text'),
'metadata': {}
}),
Interval(
Bounds3D(20, 30), {
'spatial_type':
SpatialType_Caption(
'>> this caption text will appear on a new line'),
'metadata': {}
})
])
})
vgrid_spec = VGridSpec(video_meta=video_metadata,
vis_format=VideoBlockFormat(imaps=[('bboxes', ism)]))
def test_union(self):
c= TestIntervalSetMapping.get_collection()
c1 = IntervalSetMapping({v: c[v] for v in c if v % 2 ==0})
c2 = IntervalSetMapping({v: c[v] for v in c if v % 2 ==1})
c3 = c1.union(c2)
self.assertCollectionEq(c3, c)
'''
This example shows how to draw a simple bounding box with text when expanded.
'''
# Rekall imports
from rekall import Interval, IntervalSet, IntervalSetMapping, Bounds3D
# Vgrid imports
from vgrid import VGridSpec, VideoMetadata, VideoBlockFormat, SpatialType_Bbox
# This example assumes a 1920x1080 video at 59.94 FPS, with 20,696 frames.
# You should modify it for your own examples.
video_metadata = [
VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920,
1080)
]
ism = IntervalSetMapping({
0:
IntervalSet([
Interval(Bounds3D(0, 10),
{'spatial_type': SpatialType_Bbox(text='my text')})
])
})
vgrid_spec = VGridSpec(video_meta=video_metadata,
vis_format=VideoBlockFormat(imaps=[('bboxes', ism)]))
# Pass this to your Javascript application somehow
json_for_js = vgrid_spec.to_json_compressed()
def main():
'''
An example of using IntervalSetMapping to handle annotations from multiple
videos.
'''
# Annotations for one video
is1 = IntervalSet([
Interval(Bounds3D(1, 10, 0.3, 0.4, 0.5, 0.6), 1),
Interval(Bounds3D(2, 5, 0.2, 0.8, 0.2, 0.3), 1),
Interval(Bounds3D(10, 11, 0.2, 0.7, 0.3, 0.5), 1),
Interval(Bounds3D(13, 15, 0.5, 1, 0, 0.5), 1),
Interval(Bounds3D(15, 19, 0.5, 1, 0, 0.5), 1),
Interval(Bounds3D(20, 20), payload=1),
Interval(Bounds3D(22, 22), payload=1),
Interval(Bounds3D(22, 23), payload=1),
])
# Annotations for a second video
is2 = IntervalSet([
Interval(Bounds3D(2, 5, 0.2, 0.8, 0.2, 0.4), 1),
Interval(Bounds3D(1, 10, 0.3, 0.4, 0.3, 0.6), 1),
Interval(Bounds3D(9, 11, 0.16, 0.17, 0.3, 0.5), 1),
Interval(Bounds3D(13, 15, 0.5, 1, 0, 0.5), 1),
Interval(Bounds3D(14, 19, 0.5, 1, 0, 0.5), 1),
])
# An IntervalSetMapping can store both
ism = IntervalSetMapping({1: is1, 2: is2})
# An IntervalSetMapping reflects IntervalSet operations
ism_coalesced = ism.coalesce(('t1', 't2'), Bounds3D.span, payload_plus)
other_ism = IntervalSetMapping({
1:
IntervalSet([Interval(Bounds3D(1, 5, 0, 1, 0, 1))]),
2:
IntervalSet([Interval(Bounds3D(10, 15, 0, 1, 0, 1))])
})
# For binary operations, IntervalSetMapping sends the operations to the
# correct underlying IntervalSets
isms_joined = ism.join(
other_ism, Bounds3D.T(overlaps()),
lambda i1, i2: Interval(i1['bounds'].combine_per_axis(
i2['bounds'], utils.bounds_intersect, utils.bounds_intersect, utils
.bounds_intersect)))
print('ism:')
print(ism)
print()
print('ism coalesced:')
print(ism_coalesced)
print()
print('other_ism:')
print(other_ism)
print()
print('ism joined with other_ism:')
print(isms_joined)
# This example assumes a 1920x1080 video at 59.94 FPS, with 20,696 frames.
# You should modify it for your own examples.
video_metadata = [
VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920, 1080)
]
ism = IntervalSetMapping({
0: IntervalSet([
Interval(
Bounds3D(0, 10),
{
'spatial_type': SpatialType_Temporal(),
'metadata': {
'generic_metadata': Metadata_Generic(
'this will be drawn below the timeline when expanded'
),
'other_generic_metadata': Metadata_Generic({
'key': 'this object will also be stringified'
})
}
}
)
])
})
vgrid_spec = VGridSpec(
video_meta = video_metadata,
vis_format = VideoBlockFormat(imaps = [
('bboxes', ism)
])
)
[0.0, 0.0, 0.0],
[0.0, 0.0, 0.0],
[0.2253301590681076, 0.2600831389427185, 0.30744627118110657],
[0.2345585972070694, 0.26827168464660645, 0.34357455372810364],
[0.21832072734832764, 0.271005243062973, 0.08734595775604248],
[0.24386557936668396, 0.2819412648677826, 0.30233266949653625]
]
ism = IntervalSetMapping({
0: IntervalSet([
Interval(
Bounds3D(0, 10, 0.17199, 0.24505, 0.26008, 0.46496),
{
'spatial_type': SpatialType_Keypoints(),
'metadata': {
# This function can also parse faces and hands
'pose': Metadata_Keypoints.from_openpose(
openpose_output
)
}
}
)
])
})
vgrid_spec = VGridSpec(
video_meta = video_metadata,
vis_format = VideoBlockFormat(imaps = [
('bboxes', ism)
])
)
)
)
pose_metadata[video_meta.id] = IntervalSet(pose_intervals)
# In[29]:
pose_annotation_files
# In[31]:
vgrid_spec = VGridSpec(
video_meta = metadata_videos,
vis_format = VideoBlockFormat(imaps = [
('pose', IntervalSetMapping(pose_metadata))
]),
video_endpoint = 'http://localhost:8080'
)
VGridWidget(vgrid_spec = vgrid_spec.to_json_compressed())
# In[ ]:
# Vgrid imports
from vgrid import VGridSpec, VideoMetadata, VideoBlockFormat, SpatialType_Bbox
# This example assumes a 1920x1080 video at 59.94 FPS, with 20,696 frames.
# You should modify it for your own examples.
video_metadata = [
VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920, 1080)
]
ism = IntervalSetMapping({
0: IntervalSet([
Interval(
Bounds3D(0, 10, 0.7, 0.9, 0.1, 0.8),
{
'spatial_type': SpatialType_Bbox(),
'metadata': {}
}
)
])
})
vgrid_spec = VGridSpec(
video_meta = video_metadata,
vis_format = VideoBlockFormat(imaps = [
('bboxes', ism)
])
)
# Pass this to your Javascript application somehow
json_for_js = vgrid_spec.to_json_compressed()