def test_minus(self): c= TestIntervalSetMapping.get_collection() c1 = IntervalSetMapping({v: c[v] for v in c if v % 2 ==0}) c2 = IntervalSetMapping({v: c[v] for v in c if v % 2 ==1}) c3 = c.minus(c1, window=0) self.assertCollectionEq(c3,c2)
def disjoint_domain_combiner(result1, result2): """ A faster combiner than union_combiner for IntervalSetMapping. Assumes that the results are of type IntervalSetMapping and every chunk produces its own set of domain keys that are disjoint from results of other chunks. Args: result1, result2 (IntervalSetMapping): partial results from some chunks of total work. Returns: A IntervalSetMapping that is the union of the two. Raises: RekallRuntimeException: Raised if results have common domain key. """ d1 = result1.get_grouped_intervals() d2 = result2.get_grouped_intervals() k1, k2 = set(d1.keys()), set(d2.keys()) if k1.isdisjoint(k2): return IntervalSetMapping({**d1, **d2}) intersection = k1 & k2 raise RekallRuntimeException( "DisjointDomainCombiner used on results" " with overlapping domains {0}".format(intersection))
def get_ground_truth(): interval = 30 GT_FOLDER = 'empty_spaces' empty_parking_spaces = [ pickle.loads( requests.get(posixpath.join( posixpath.join(VIDEO_COLLECTION_BASEURL, GT_FOLDER), posixpath.join(vm.path[:-4], 'gt.pkl')), verify=False).content) for vm in video_metadata ] gt_ism = IntervalSetMapping({ metadata.id: IntervalSet([ Interval( Bounds3D(t1=30 * i / metadata.fps, t2=30 * (i + interval) / metadata.fps, x1=bbox[0] / metadata.width + .01, x2=bbox[2] / metadata.width - .01, y1=bbox[1] / metadata.height + .01, y2=bbox[3] / metadata.height - .01)) for i, frame in enumerate(space_frame_list) if (i % interval == 0) for bbox in frame ]) for space_frame_list, metadata in tqdm(zip(empty_parking_spaces, video_metadata), total=len(empty_parking_spaces)) }) return gt_ism
def get_maskrcnn_bboxes(): interval = 30 bboxes = [ pickle.loads( requests.get(posixpath.join( posixpath.join(VIDEO_COLLECTION_BASEURL, BBOX_FOLDER), posixpath.join(vm.path[:-4], 'bboxes.pkl')), verify=False).content) for vm in (video_metadata) ] bboxes_ism = IntervalSetMapping({ metadata.id: IntervalSet([ Interval(Bounds3D(t1=30 * i / metadata.fps, t2=30 * (i + interval) / metadata.fps, x1=bbox[0] / metadata.width, x2=bbox[2] / metadata.width, y1=bbox[1] / metadata.height, y2=bbox[3] / metadata.height), payload={ 'class': bbox[4], 'score': bbox[5], 'spatial_type': SpatialType_Bbox(text=bbox[4]) }) for i, frame in enumerate(bbox_frame_list) if (i % interval == 0) for bbox in frame ]) for bbox_frame_list, metadata in tqdm(zip(bboxes, (video_metadata)), total=len(bboxes)) }) return bboxes_ism
def test_collect_by_interval(self): c = TestIntervalSetMapping.get_collection() d = IntervalSetMapping({1: IntervalSet([ Interval(Bounds3D(t,t)) for t in range(1, 100)])}) e = c.collect_by_interval(d, Bounds3D.T(overlaps()), filter_empty=False, window=0) self.assertEqual(e.keys(), c.keys())
def list_to_IntervalSetMapping(interval_list): ism = {} for video_id, start, end, duration in interval_list: if not video_id in ism: ism[video_id] = [] ism[video_id].append(Interval(Bounds3D(start, end))) return IntervalSetMapping({video_id: IntervalSet(intervalSet) for video_id, intervalSet in ism.items()})
def IntervalSetMapping_second_to_frame(ism): intervalSets_frame = {} for video_id, intervalSet in ism.get_grouped_intervals().items(): video = Video.objects.filter(id=video_id)[0] fps = video.fps intervalSets_frame[video_id] = IntervalSet( [Interval(Bounds1D(int(i.bounds['t1'] * fps), int(i.bounds['t2'] * fps)), i.payload) \ for i in intervalSet.get_intervals()] ) return IntervalSetMapping(intervalSets_frame)
def get_collection(): NUM_INTERVALS=150 NUM_VIDEOS=100 def get_intervals_for_video(vid): intervals = [ Interval(Bounds3D(t, t+1,0,0.5,0.5,1),vid) for t in range(vid, vid+NUM_INTERVALS)] return IntervalSet(intervals) return IntervalSetMapping( {vid: get_intervals_for_video(vid) for vid in range( NUM_VIDEOS)})
def all_videos(): from scripts.models import Video from vgrid import VideoVBlocksBuilder, VideoTrackBuilder, VideoMetadata from rekall import IntervalSet, Interval, Bounds3D, IntervalSetMapping videos = Video.objects.all() json = VideoVBlocksBuilder() \ .add_track(VideoTrackBuilder( 'videos', IntervalSetMapping({v.id: IntervalSet([Interval(Bounds3D(0, v.duration()))]) for v in videos}))) \ .add_video_metadata( '/system_media', [VideoMetadata(**{k: getattr(v, k) for k in ['id', 'path', 'num_frames', 'fps', 'width', 'height']}) for v in videos]) \ .build() return json
def rs_to_rekall(rs_ilist, video_ids=None, with_payload=True): rekall_ism = {} if video_ids is None: video_ids = rs_ilist.get_ids() for video_id in video_ids: if with_payload: interval_list = rs_ilist.get_intervals_with_payload(video_id, True) rekall_ism[video_id] = IntervalSet([ Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0), i[2]) for i in interval_list ]) else: interval_list = rs_ilist.get_intervals(video_id, True) rekall_ism[video_id] = IntervalSet([ Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0)) for i in interval_list ]) return IntervalSetMapping(rekall_ism)
def get_maskrcnn_bboxes(): maskrcnn_bboxes_ism = IntervalSetMapping({ vm.id: IntervalSet([ Interval(Bounds3D(t1=frame_num / vm.fps, t2=(frame_num + 1) / vm.fps, x1=bbox[0] / vm.width, x2=bbox[2] / vm.width, y1=bbox[1] / vm.height, y2=bbox[3] / vm.height), payload={ 'class': bbox[4], 'score': bbox[5] }) for frame_num, bboxes_in_frame in enumerate(maskrcnn_frame_list) for bbox in bboxes_in_frame ]) for vm, maskrcnn_frame_list in zip(video_metadata_intel, maskrcnn_bboxes_intel) }) return maskrcnn_bboxes_ism
# Rekall imports from rekall import Interval, IntervalSet, IntervalSetMapping, Bounds3D # Vgrid imports from vgrid import VGridSpec, VideoMetadata, VideoBlockFormat from vgrid import SpatialType_Temporal # This example assumes a 1920x1080 video at 59.94 FPS, with 20,696 frames. # You should modify it for your own examples. video_metadata = [ VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920, 1080) ] ism = IntervalSetMapping({ 0: IntervalSet([ Interval(Bounds3D(0, 10), { 'spatial_type': SpatialType_Temporal(), 'metadata': {} }) ]) }) vgrid_spec = VGridSpec(video_meta=video_metadata, vis_format=VideoBlockFormat(imaps=[('bboxes', ism)])) # Pass this to your Javascript application somehow json_for_js = vgrid_spec.to_json_compressed()
def query(vids): return IntervalSetMapping( {vid: TestRuntime.dummy_intervalset() for vid in vids})
VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920, 1080) ] ism = IntervalSetMapping({ 0: IntervalSet([ Interval( Bounds3D(0, 10), { 'spatial_type': SpatialType_Caption('this is caption text'), 'metadata': {} }), Interval( Bounds3D(10, 20), { 'spatial_type': SpatialType_Caption('this is more caption text'), 'metadata': {} }), Interval( Bounds3D(20, 30), { 'spatial_type': SpatialType_Caption( '>> this caption text will appear on a new line'), 'metadata': {} }) ]) }) vgrid_spec = VGridSpec(video_meta=video_metadata, vis_format=VideoBlockFormat(imaps=[('bboxes', ism)]))
def test_union(self): c= TestIntervalSetMapping.get_collection() c1 = IntervalSetMapping({v: c[v] for v in c if v % 2 ==0}) c2 = IntervalSetMapping({v: c[v] for v in c if v % 2 ==1}) c3 = c1.union(c2) self.assertCollectionEq(c3, c)
''' This example shows how to draw a simple bounding box with text when expanded. ''' # Rekall imports from rekall import Interval, IntervalSet, IntervalSetMapping, Bounds3D # Vgrid imports from vgrid import VGridSpec, VideoMetadata, VideoBlockFormat, SpatialType_Bbox # This example assumes a 1920x1080 video at 59.94 FPS, with 20,696 frames. # You should modify it for your own examples. video_metadata = [ VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920, 1080) ] ism = IntervalSetMapping({ 0: IntervalSet([ Interval(Bounds3D(0, 10), {'spatial_type': SpatialType_Bbox(text='my text')}) ]) }) vgrid_spec = VGridSpec(video_meta=video_metadata, vis_format=VideoBlockFormat(imaps=[('bboxes', ism)])) # Pass this to your Javascript application somehow json_for_js = vgrid_spec.to_json_compressed()
def main(): ''' An example of using IntervalSetMapping to handle annotations from multiple videos. ''' # Annotations for one video is1 = IntervalSet([ Interval(Bounds3D(1, 10, 0.3, 0.4, 0.5, 0.6), 1), Interval(Bounds3D(2, 5, 0.2, 0.8, 0.2, 0.3), 1), Interval(Bounds3D(10, 11, 0.2, 0.7, 0.3, 0.5), 1), Interval(Bounds3D(13, 15, 0.5, 1, 0, 0.5), 1), Interval(Bounds3D(15, 19, 0.5, 1, 0, 0.5), 1), Interval(Bounds3D(20, 20), payload=1), Interval(Bounds3D(22, 22), payload=1), Interval(Bounds3D(22, 23), payload=1), ]) # Annotations for a second video is2 = IntervalSet([ Interval(Bounds3D(2, 5, 0.2, 0.8, 0.2, 0.4), 1), Interval(Bounds3D(1, 10, 0.3, 0.4, 0.3, 0.6), 1), Interval(Bounds3D(9, 11, 0.16, 0.17, 0.3, 0.5), 1), Interval(Bounds3D(13, 15, 0.5, 1, 0, 0.5), 1), Interval(Bounds3D(14, 19, 0.5, 1, 0, 0.5), 1), ]) # An IntervalSetMapping can store both ism = IntervalSetMapping({1: is1, 2: is2}) # An IntervalSetMapping reflects IntervalSet operations ism_coalesced = ism.coalesce(('t1', 't2'), Bounds3D.span, payload_plus) other_ism = IntervalSetMapping({ 1: IntervalSet([Interval(Bounds3D(1, 5, 0, 1, 0, 1))]), 2: IntervalSet([Interval(Bounds3D(10, 15, 0, 1, 0, 1))]) }) # For binary operations, IntervalSetMapping sends the operations to the # correct underlying IntervalSets isms_joined = ism.join( other_ism, Bounds3D.T(overlaps()), lambda i1, i2: Interval(i1['bounds'].combine_per_axis( i2['bounds'], utils.bounds_intersect, utils.bounds_intersect, utils .bounds_intersect))) print('ism:') print(ism) print() print('ism coalesced:') print(ism_coalesced) print() print('other_ism:') print(other_ism) print() print('ism joined with other_ism:') print(isms_joined)
# This example assumes a 1920x1080 video at 59.94 FPS, with 20,696 frames. # You should modify it for your own examples. video_metadata = [ VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920, 1080) ] ism = IntervalSetMapping({ 0: IntervalSet([ Interval( Bounds3D(0, 10), { 'spatial_type': SpatialType_Temporal(), 'metadata': { 'generic_metadata': Metadata_Generic( 'this will be drawn below the timeline when expanded' ), 'other_generic_metadata': Metadata_Generic({ 'key': 'this object will also be stringified' }) } } ) ]) }) vgrid_spec = VGridSpec( video_meta = video_metadata, vis_format = VideoBlockFormat(imaps = [ ('bboxes', ism) ]) )
[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.2253301590681076, 0.2600831389427185, 0.30744627118110657], [0.2345585972070694, 0.26827168464660645, 0.34357455372810364], [0.21832072734832764, 0.271005243062973, 0.08734595775604248], [0.24386557936668396, 0.2819412648677826, 0.30233266949653625] ] ism = IntervalSetMapping({ 0: IntervalSet([ Interval( Bounds3D(0, 10, 0.17199, 0.24505, 0.26008, 0.46496), { 'spatial_type': SpatialType_Keypoints(), 'metadata': { # This function can also parse faces and hands 'pose': Metadata_Keypoints.from_openpose( openpose_output ) } } ) ]) }) vgrid_spec = VGridSpec( video_meta = video_metadata, vis_format = VideoBlockFormat(imaps = [ ('bboxes', ism) ]) )
) ) pose_metadata[video_meta.id] = IntervalSet(pose_intervals) # In[29]: pose_annotation_files # In[31]: vgrid_spec = VGridSpec( video_meta = metadata_videos, vis_format = VideoBlockFormat(imaps = [ ('pose', IntervalSetMapping(pose_metadata)) ]), video_endpoint = 'http://localhost:8080' ) VGridWidget(vgrid_spec = vgrid_spec.to_json_compressed()) # In[ ]:
# Vgrid imports from vgrid import VGridSpec, VideoMetadata, VideoBlockFormat, SpatialType_Bbox # This example assumes a 1920x1080 video at 59.94 FPS, with 20,696 frames. # You should modify it for your own examples. video_metadata = [ VideoMetadata('http://localhost:8000/test.mp4', 0, 59.94, 20696, 1920, 1080) ] ism = IntervalSetMapping({ 0: IntervalSet([ Interval( Bounds3D(0, 10, 0.7, 0.9, 0.1, 0.8), { 'spatial_type': SpatialType_Bbox(), 'metadata': {} } ) ]) }) vgrid_spec = VGridSpec( video_meta = video_metadata, vis_format = VideoBlockFormat(imaps = [ ('bboxes', ism) ]) ) # Pass this to your Javascript application somehow json_for_js = vgrid_spec.to_json_compressed()