def test_match_multiple_solutions(self): left_box_frame_1 = Interval(Bounds3D(1, 1, 0.1, 0.4, 0.4, 0.8)) right_box_frame_1 = Interval(Bounds3D(1, 1, 0.6, 0.9, 0.3, 0.7)) bottom_left_box_frame_2 = Interval(Bounds3D(2, 2, 0.1, 0.3, 0.8, 0.9)) top_right_box_frame_2 = Interval(Bounds3D(2, 2, 0.5, 0.7, 0.2, 0.7)) is1 = IntervalSet([ left_box_frame_1, right_box_frame_1, bottom_left_box_frame_2, top_right_box_frame_2, Interval(Bounds3D(3, 3)), ]) pattern = [ ( ["left", "right"], [ # Two boxes on left and right on same frame Bounds3D.T(equal()), Bounds3D.XY(left_of()), ]), ] results = is1.match(pattern, exact=False) self.assertEqual(len(results), 2) # Add single interval constraints. pattern = pattern + [ (["left"], [Bounds3D.XY(height_at_least(0.3))]), (["right"], [Bounds3D.XY(height_at_least(0.3))]), ] results = is1.match(pattern, exact=False) self.assertEqual(len(results), 1) result = results[0] self.assertIntervalsEq(left_box_frame_1, result['left']) self.assertIntervalsEq(right_box_frame_1, result['right'])
def test_group_by(self): is1 = IntervalSet([ Interval(Bounds3D(0, 1)), Interval(Bounds3D(1, 2)), Interval(Bounds3D(0, 2)), Interval(Bounds3D(1, 3)), ]) def merge_intervals(k, intervals): merged = intervals.fold( lambda i1, i2: Interval(i1['bounds'].span(i2['bounds']))) merged['payload'] = intervals return merged is2 = is1.group_by(lambda i: int(i['t2']) % 2, merge_intervals) target = IntervalSet([ Interval(Bounds3D(0, 3), payload=IntervalSet([ Interval(Bounds3D(0, 1)), Interval(Bounds3D(1, 3)), ])), Interval(Bounds3D(0, 2), payload=IntervalSet([ Interval(Bounds3D(0, 2)), Interval(Bounds3D(1, 2)), ])), ]) self.assertIntervalSetEq(is2, target, self.compare_interval_sets_in_payload())
def test_len(self): is1 = IntervalSet([ Interval(Bounds3D(1, 5)), Interval(Bounds3D(10, 22)), ]) self.assertEqual(len(is1), 2) self.assertEqual(is1.size(), 2)
def process_single(black_frames_path, captions_path, metadata_path, commercials_outpath): # Load original data black_frames = load_json(black_frames_path) captions = load_captions(captions_path) metadata = load_json(metadata_path) # Create IntervalSet objects black_frames_set = IntervalSet([ Interval( Bounds3D(frame_num / metadata['fps'], (frame_num + 1) / metadata['fps'])) for frame_num in black_frames ]) captions_set = IntervalSet([ Interval(Bounds3D(start, end), payload=text) for text, start, end, in captions ]) whole_video = IntervalSet( [Interval(Bounds3D(0, metadata['frames'] / metadata['fps']))]) # Detect commercials results = detect_commercials(black_frames_set, captions_set, whole_video) # Convert commercial intervals back to frames results = convert_set_from_seconds_to_frames(results, metadata['fps']) # Save results in JSON format results = [(r['t1'], r['t2']) for r in results.get_intervals()] save_json(results, commercials_outpath)
def fold_fn(stack, interval): if interval['t2'] - interval['t1'] > MAX_COMMERCIAL_TIME: interval = Interval( Bounds3D(interval['t1'], interval['t1'] + MAX_COMMERCIAL_TIME)) if len(stack) == 0: stack.append(interval) else: last = stack.pop() if or_pred(overlaps(), after(max_dist=COMMERCIAL_FOLD_EPSILON))(interval, last): if last['bounds'].span(interval['bounds']).size() \ > MAX_COMMERCIAL_TIME: stack.append( Interval( Bounds3D(last['t1'], last['t1'] + MAX_COMMERCIAL_TIME))) else: stack.append( Interval(last['bounds'].span(interval['bounds']))) else: stack.append(last) stack.append(interval) return stack
def main(): ''' An example of a join. We filter pairs based on time overlap, and merge surviving pairs by taking the intersection over the time bounds and the span of the spatial bounds (and adding the payloads). ''' is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2), ]) is2 = IntervalSet([ Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 4), Interval(Bounds3D(0, 1, 0, 1, 0, 1), 8), ]) is3 = is1.join( is2, Bounds3D.T(overlaps()), lambda i1, i2: Interval( i1['bounds'].intersect_time_span_space(i2['bounds']), i1['payload'] + i2['payload'])) print('is1:') print(is1) print('is2:') print(is2) print('is1 joined with is2:') print(is3)
def test_fold(self): def fold_fn(acc, i): return acc + (i['bounds'].length() * i['bounds'].width() * i['bounds'].height()) is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2), Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3), ]) self.assertAlmostEqual(is1.fold(fold_fn, 0), 1.375)
def test_filter(self): is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2), Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3), ]) is1 = is1.filter(lambda i: i['payload'] > 2) target = IntervalSet([ Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3), ]) self.assertIntervalSetEq(is1, target, eq)
def test_join_with_optimization_window(self): is1 = IntervalSet( [Interval(Bounds3D(t, t + 1), t) for t in range(100)]) is2 = IntervalSet([Interval(Bounds3D(t, t), t) for t in range(100)]) is3 = is1.join(is2, Bounds3D.T(before(max_dist=1)), lambda i1, i2: Interval(i1['bounds'].span(i2['bounds']), i2['payload']), window=1) target = IntervalSet( [Interval(Bounds3D(t, t + 2), t + 2) for t in range(98)] + [Interval(Bounds3D(t, t + 1), t + 1) for t in range(99)]) self.assertIntervalSetEq(is3, target, eq)
def test_fold_custom_sortkey(self): def sortkey(i): return (i['x1'], i['x2']) def fold_fn(acc, i): acc.append(i['payload']) return acc is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 0.1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2), Interval(Bounds3D(0.1, 0.4, 0, 1, 0, 0.5), 3), ]) self.assertListEqual(is1.fold(fold_fn, [], sortkey), [1, 3, 2])
def expand_to_frame(intrvl): new_bounds = intrvl['bounds'].copy() new_bounds['x1'] = 0 new_bounds['x2'] = 1 new_bounds['y1'] = 0 new_bounds['y2'] = 1 return Interval(new_bounds, intrvl['payload'])
def test_collect_by_interval(self): c = TestIntervalSetMapping.get_collection() d = IntervalSetMapping({1: IntervalSet([ Interval(Bounds3D(t,t)) for t in range(1, 100)])}) e = c.collect_by_interval(d, Bounds3D.T(overlaps()), filter_empty=False, window=0) self.assertEqual(e.keys(), c.keys())
def find_clips_for_keyword(keyword, use_only_video=False): ism = {} for vm in tqdm(video_metadata): if not vm["annotation_filename"] or (not vm["only_video"] and use_only_video): continue try: h5 = eeghdf.Eeghdf(vm["annotation_filename"]) except: print(vm["annotation_filename"]) os.remove(vm["annotation_filename"]) continue starts = [start / 10**7 for start in h5._annotation_start100ns] texts = h5._annotation_text if not keyword or any(keyword.lower() in text.lower() for text in texts): interval_set = IntervalSet([ Interval( Bounds3D(start, start + 5), # we set the duration { 'spatial_type': SpatialType_Caption(">>" + text + "\n"), 'metadata': {} }) for start, text in zip(starts, texts) ]) ism[vm["id"]] = interval_set print( f"Found {len(ism)} videos with keyword {keyword} in the annotations.") vgrid_spec = VGridSpec(video_meta=video_metadata_wrapper, vis_format=VideoBlockFormat(imaps=[('bboxes', ism)]), video_endpoint='http://localhost:8080') return VGridWidget(vgrid_spec=vgrid_spec.to_json_compressed())
def list_to_IntervalSetMapping(interval_list): ism = {} for video_id, start, end, duration in interval_list: if not video_id in ism: ism[video_id] = [] ism[video_id].append(Interval(Bounds3D(start, end))) return IntervalSetMapping({video_id: IntervalSet(intervalSet) for video_id, intervalSet in ism.items()})
def get_ground_truth(): interval = 30 GT_FOLDER = 'empty_spaces' empty_parking_spaces = [ pickle.loads( requests.get(posixpath.join( posixpath.join(VIDEO_COLLECTION_BASEURL, GT_FOLDER), posixpath.join(vm.path[:-4], 'gt.pkl')), verify=False).content) for vm in video_metadata ] gt_ism = IntervalSetMapping({ metadata.id: IntervalSet([ Interval( Bounds3D(t1=30 * i / metadata.fps, t2=30 * (i + interval) / metadata.fps, x1=bbox[0] / metadata.width + .01, x2=bbox[2] / metadata.width - .01, y1=bbox[1] / metadata.height + .01, y2=bbox[3] / metadata.height - .01)) for i, frame in enumerate(space_frame_list) if (i % interval == 0) for bbox in frame ]) for space_frame_list, metadata in tqdm(zip(empty_parking_spaces, video_metadata), total=len(empty_parking_spaces)) }) return gt_ism
def get_maskrcnn_bboxes(): interval = 30 bboxes = [ pickle.loads( requests.get(posixpath.join( posixpath.join(VIDEO_COLLECTION_BASEURL, BBOX_FOLDER), posixpath.join(vm.path[:-4], 'bboxes.pkl')), verify=False).content) for vm in (video_metadata) ] bboxes_ism = IntervalSetMapping({ metadata.id: IntervalSet([ Interval(Bounds3D(t1=30 * i / metadata.fps, t2=30 * (i + interval) / metadata.fps, x1=bbox[0] / metadata.width, x2=bbox[2] / metadata.width, y1=bbox[1] / metadata.height, y2=bbox[3] / metadata.height), payload={ 'class': bbox[4], 'score': bbox[5], 'spatial_type': SpatialType_Bbox(text=bbox[4]) }) for i, frame in enumerate(bbox_frame_list) if (i % interval == 0) for bbox in frame ]) for bbox_frame_list, metadata in tqdm(zip(bboxes, (video_metadata)), total=len(bboxes)) }) return bboxes_ism
def IntervalSetMapping_second_to_frame(ism): intervalSets_frame = {} for video_id, intervalSet in ism.get_grouped_intervals().items(): video = Video.objects.filter(id=video_id)[0] fps = video.fps intervalSets_frame[video_id] = IntervalSet( [Interval(Bounds1D(int(i.bounds['t1'] * fps), int(i.bounds['t2'] * fps)), i.payload) \ for i in intervalSet.get_intervals()] ) return IntervalSetMapping(intervalSets_frame)
def split_fn(i): output = [] t = i.copy() while t['bounds'].length() > 5: output.append(Interval(Bounds3D(t['t1'], t['t1'] + 5))) t['t1'] = t['t1'] + 5 if t['bounds'].length() > 0: output.append(t) return IntervalSet(output)
def test_map(self): def expand_to_frame(intrvl): new_bounds = intrvl['bounds'].copy() new_bounds['x1'] = 0 new_bounds['x2'] = 1 new_bounds['y1'] = 0 new_bounds['y2'] = 1 return Interval(new_bounds, intrvl['payload']) is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0.3, 0.4, 0.5, 0.6)), Interval(Bounds3D(0, 0.5, 0.2, 0.3, 0.5, 0.6)) ]) is1 = is1.map(expand_to_frame) target = IntervalSet([ Interval(Bounds3D(0, 0.5, 0, 1, 0, 1)), Interval(Bounds3D(0, 1, 0, 1, 0, 1)), ]) self.assertIntervalSetEq(is1, target)
def rs_to_rekall(rs_ilist, video_ids=None, with_payload=True): rekall_ism = {} if video_ids is None: video_ids = rs_ilist.get_ids() for video_id in video_ids: if with_payload: interval_list = rs_ilist.get_intervals_with_payload(video_id, True) rekall_ism[video_id] = IntervalSet([ Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0), i[2]) for i in interval_list ]) else: interval_list = rs_ilist.get_intervals(video_id, True) rekall_ism[video_id] = IntervalSet([ Interval(Bounds3D(i[0] / 1000., i[1] / 1000., 0, 0, 0, 0)) for i in interval_list ]) return IntervalSetMapping(rekall_ism)
def for_vgrid(self): fps = self.obj.video.fps pad = 0.02 return IntervalSet([ Interval( Bounds3D(t1=p.frame / fps, t2=(p.frame + 1) / fps, x1=p.to_pixel_norm()[0] - pad, x2=p.to_pixel_norm()[0] + pad, y1=p.to_pixel_norm()[1] - pad, y2=p.to_pixel_norm()[1] + pad)) for p in self.pos ])
def test_minus_everything(self): is1 = IntervalSet( [Interval(Bounds3D(1, 10)), Interval(Bounds3D(3, 15))]) is2 = IntervalSet([ Interval(Bounds3D(2, 2.5)), Interval(Bounds3D(2, 2.7)), Interval(Bounds3D(2.9, 3.5)), Interval(Bounds3D(3.5, 3.6)), Interval(Bounds3D(5, 7)), Interval(Bounds3D(9, 12)), ]) is3 = is2.minus(is1) self.assertIntervalSetEq(is3, IntervalSet([]), eq)
def __init__(self, block_label_state): self._new_intervals = IntervalSet([ Interval( Bounds3D(t1=intvl['bounds']['t1'], t2=intvl['bounds']['t2'], x1=intvl['bounds']['bbox']['x1'], x2=intvl['bounds']['bbox']['x2'], y1=intvl['bounds']['bbox']['y1'], y2=intvl['bounds']['bbox']['y2'])) for intvl in block_label_state['new_intervals'] ]) self._captions_selected = None # TODO
def test_minus_with_single_frame(self): is1 = IntervalSet([ Interval(Bounds3D(1, 1)), Interval(Bounds3D(3, 3)), Interval(Bounds3D(4, 4)), Interval(Bounds3D(7, 7)), Interval(Bounds3D(10, 10)), ]) is2 = IntervalSet([ Interval(Bounds3D(1, 3)), Interval(Bounds3D(5, 8)), Interval(Bounds3D(9, 9)), ]) is3 = is1.minus(is2) target = IntervalSet([ Interval(Bounds3D(4, 4)), Interval(Bounds3D(10, 10)), ]) self.assertIntervalSetEq(is3, target) is4 = is2.minus(is1) self.assertIntervalSetEq(is4, is2)
def test_match(self): left_box_frame_1 = Interval(Bounds3D(1, 1, 0.1, 0.4, 0.4, 0.8)) right_box_frame_1 = Interval(Bounds3D(1, 1, 0.6, 0.9, 0.3, 0.7)) bottom_left_box_frame_2 = Interval(Bounds3D(2, 2, 0.1, 0.3, 0.8, 0.9)) top_right_box_frame_2 = Interval(Bounds3D(2, 2, 0.5, 0.7, 0.2, 0.7)) is1 = IntervalSet([ left_box_frame_1, right_box_frame_1, bottom_left_box_frame_2, top_right_box_frame_2 ]) pattern = [ ( ["left", "right"], [ # Two boxes on left and right on same frame Bounds3D.T(equal()), Bounds3D.XY(left_of()), ]), ( ["top", "bottom"], [ # Two boxes on top and bottom on overlapping frame Bounds3D.T(equal()), Bounds3D.XY(above()), ]), ( ["left", "top"], [ # Left-Right pattern comes before Top-Bottom Bounds3D.T(meets_before(epsilon=1)) ]) ] results = is1.match(pattern, exact=True) self.assertEqual(len(results), 1) result = results[0] self.assertIntervalsEq(left_box_frame_1, result['left']) self.assertIntervalsEq(right_box_frame_1, result['right']) self.assertIntervalsEq(top_right_box_frame_2, result['top']) self.assertIntervalsEq(bottom_left_box_frame_2, result['bottom'])
def test_minus_against_nothing(self): is1 = IntervalSet([ Interval(Bounds3D(1, 10, 0, 0.5, 0.2, 0.8), 1), Interval(Bounds3D(3, 15, 0, 1, 0, 1), 2) ]) is2 = IntervalSet([ Interval(Bounds3D(20, 20.5)), Interval(Bounds3D(20, 20.7)), Interval(Bounds3D(20.9, 23.5)), Interval(Bounds3D(23.5, 23.6)), Interval(Bounds3D(25, 27)), Interval(Bounds3D(29, 32)), ]) is3 = is1.minus(is2) self.assertIntervalSetEq(is3, is1, eq)
def main(): ''' Examples of creating some Intervals. Intervals are parameterized by Bounds (Bounds3D in these examples) and an optional payload. ''' my_first_bounds = Bounds3D(t1=5, t2=10, x1=0.1, x2=0.4, y1=0.3, y2=0.9) # This bounds is the same as my_first_bounds my_second_bounds = Bounds3D(5, 10, 0.1, 0.4, 0.3, 0.9) # This Interval has the 3D bounds of my_first_bounds and a payload my_first_interval = Interval(my_first_bounds, payload=10) # This Interval doesn't have a payload my_first_interval_no_payload = Interval(my_first_bounds) # IntervalSet contains multiple Intervals my_first_interval_set = IntervalSet([ my_first_interval, Interval(my_second_bounds, payload='payloads are arbitrary'), Interval(Bounds3D(20, 30), payload='Bounds3D has default X/Y fields') ]) print(my_first_interval_set)
def main(): ''' Examples of simple functions on IntervalSets. ''' # filter examples is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2), Interval(Bounds3D(0, 0.5, 0, 1, 0, 0.5), 3), ]) print('is1:') print(is1) is1_filtered = is1.filter(lambda i: i['payload'] > 2) print('is1 filtered on payload:') print(is1_filtered) print() # map example def expand_to_frame(intrvl): new_bounds = intrvl['bounds'].copy() new_bounds['x1'] = 0 new_bounds['x2'] = 1 new_bounds['y1'] = 0 new_bounds['y2'] = 1 return Interval(new_bounds, intrvl['payload']) is2 = IntervalSet([ Interval(Bounds3D(0, 1, 0.3, 0.4, 0.5, 0.6)), Interval(Bounds3D(0, 0.5, 0.2, 0.3, 0.5, 0.6)) ]) print('is2:') print(is2) is2_expanded = is2.map(expand_to_frame) print('is2 expanded to the frame:') print(is2_expanded)
def test_union(self): is1 = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 1), ]) is2 = IntervalSet([ Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 2), Interval(Bounds3D(0, 1, 0, 1, 0, 1), 2), ]) is3 = is1.union(is2) target = IntervalSet([ Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1), Interval(Bounds3D(0, 1, 0, 1, 0, 1), 2), Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 1), Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 2), ]) self.assertIntervalSetEq(is3, target, eq)
def test_minus_self(self): is1 = IntervalSet([ Interval(Bounds3D(2, 2.5)), Interval(Bounds3D(2, 2.7)), Interval(Bounds3D(2.9, 3.5)), Interval(Bounds3D(3.5, 3.6)), Interval(Bounds3D(5, 7)), Interval(Bounds3D(9, 12)), ]) is1 = is1.minus(is1) self.assertIntervalSetEq(is1, IntervalSet([]), eq)