def test_bounds_inheritance(self):
from rekall.predicates import overlaps
class Bounds2D(Bounds):
def __init__(self, t1, t2, x1, x2):
self.data = {'t1': t1, 't2': t2, 'x1': x1, 'x2': x2}
def __lt__(self, other):
return ((self['t1'], self['t2'], self['x1'], self['x2']) <
(other['t1'], other['t2'], other['x1'], other['x2']))
def primary_axis(self):
return ('t1', 't2')
def X(pred):
return Bounds.cast({'t1': 'x1', 't2': 'x2'})(pred)
bounds1 = Bounds2D(0, 1, 0.5, 0.7)
bounds2 = Bounds2D(2, 3, 0.4, 0.6)
# overlaps expects two objects with fields 't1' and 't2' and
# computes whether there is overlap in that dimension
# This is False, since there is no time overlap
self.assertFalse(overlaps()(bounds1, bounds2))
# This is True, since there is overlap in the X dimension
self.assertTrue(Bounds2D.X(overlaps())(bounds1, bounds2))
# This is True.
self.assertTrue(bounds1 < bounds2)
# This returns ('t1', 't2')
self.assertEqual(bounds1.primary_axis(), ('t1', 't2'))
def main():
'''
An example of a join. We filter pairs based on time overlap, and merge
surviving pairs by taking the intersection over the time bounds and the
span of the spatial bounds (and adding the payloads).
'''
is1 = IntervalSet([
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 1),
Interval(Bounds3D(0, 0.5, 0.5, 1, 0, 0.5), 2),
])
is2 = IntervalSet([
Interval(Bounds3D(0.5, 1, 0, 1, 0, 1), 4),
Interval(Bounds3D(0, 1, 0, 1, 0, 1), 8),
])
is3 = is1.join(
is2, Bounds3D.T(overlaps()), lambda i1, i2: Interval(
i1['bounds'].intersect_time_span_space(i2['bounds']), i1['payload']
+ i2['payload']))
print('is1:')
print(is1)
print('is2:')
print(is2)
print('is1 joined with is2:')
print(is3)
def test_collect_by_interval(self):
c = TestIntervalSetMapping.get_collection()
d = IntervalSetMapping({1: IntervalSet([
Interval(Bounds3D(t,t)) for t in range(1, 100)])})
e = c.collect_by_interval(d, Bounds3D.T(overlaps()),
filter_empty=False, window=0)
self.assertEqual(e.keys(), c.keys())
def fold_fn(stack, interval):
if interval['t2'] - interval['t1'] > MAX_COMMERCIAL_TIME:
interval = Interval(
Bounds3D(interval['t1'], interval['t1'] + MAX_COMMERCIAL_TIME))
if len(stack) == 0:
stack.append(interval)
else:
last = stack.pop()
if or_pred(overlaps(),
after(max_dist=COMMERCIAL_FOLD_EPSILON))(interval,
last):
if last['bounds'].span(interval['bounds']).size() \
> MAX_COMMERCIAL_TIME:
stack.append(
Interval(
Bounds3D(last['t1'],
last['t1'] + MAX_COMMERCIAL_TIME)))
else:
stack.append(
Interval(last['bounds'].span(interval['bounds'])))
else:
stack.append(last)
stack.append(interval)
return stack
def tiou(box1, box2) -> float:
# IoU in t1, t2
if not overlaps()(box1, box2):
return 0
U = max(box1['t2'], box2['t2']) - min(box1['t1'], box2['t1'])
I = (box1['t2'] - box1['t1']) + (box2['t2'] - box2['t1']) - U
assert U >= 0 and I >= 0
return I / U
def intersect(self, other):
"""Returns the bound intersecting ``self`` and ``other``, or
``None`` if the bounds do not overlap.
Returns:
A single Bounds1D covering the intersection of ``self`` and
``other``, or ``None`` if the two bounds do not overlap.
"""
if overlaps()(self, other):
return Bounds1D.fromTuple(
utils.bounds_intersect((self['t1'], self['t2']),
(other['t1'], other['t2'])))
else:
return None
def intersect_time_span_space(self, other):
"""Returns the bound intersecting ``other`` in time and spanning
``self`` and ``other`` in space. Returns ``None`` if ``self`` and
``other`` do not overlap in time.
Returns:
A single Bounds3D at the intersection of ``self`` and ``other`` in
time but spanning them in space, or ``None`` if they do not
overlap in time.
"""
if overlaps()(self, other):
return self.combine_per_axis(other, utils.bounds_intersect,
utils.bounds_span, utils.bounds_span)
else:
return None
def main():
'''
Some examples of coalesce -- recursively merging overlapping or touching
intervals.
'''
is1 = IntervalSet([
Interval(Bounds3D(1, 10, 0.3, 0.4, 0.5, 0.6), 1),
Interval(Bounds3D(2, 5, 0.2, 0.8, 0.2, 0.3), 1),
Interval(Bounds3D(10, 11, 0.2, 0.7, 0.3, 0.5), 1),
Interval(Bounds3D(13, 15, 0.5, 1, 0, 0.5), 1),
Interval(Bounds3D(15, 19, 0.5, 1, 0, 0.5), 1),
Interval(Bounds3D(20, 20), payload=1),
Interval(Bounds3D(22, 22), payload=1),
Interval(Bounds3D(22, 23), payload=1),
])
is1_coalesced = is1.coalesce(('t1', 't2'), Bounds3D.span, payload_plus)
# target1 is the same as is1_coalesced
target1 = IntervalSet([
Interval(Bounds3D(1, 11, 0.2, 0.8, 0.2, 0.6), 3),
Interval(Bounds3D(13, 19, 0.5, 1, 0, 0.5), 2),
Interval(Bounds3D(20, 20), payload=1),
Interval(Bounds3D(22, 23), payload=2),
])
is1_coalesced_larger_epsilon = is1.coalesce(('t1', 't2'),
Bounds3D.span,
payload_plus,
epsilon=2)
# target2 is the same as is1_coalesced_larger_epsilon
target2 = IntervalSet([Interval(Bounds3D(1, 23), payload=8)])
is2 = IntervalSet([
Interval(Bounds3D(2, 5, 0.2, 0.8, 0.2, 0.4), 1),
Interval(Bounds3D(1, 10, 0.3, 0.4, 0.3, 0.6), 1),
Interval(Bounds3D(9, 11, 0.16, 0.17, 0.3, 0.5), 1),
Interval(Bounds3D(13, 15, 0.5, 1, 0, 0.5), 1),
Interval(Bounds3D(14, 19, 0.5, 1, 0, 0.5), 1),
])
is2_coalesced_overlapping_bboxes = is2.coalesce(
('t1', 't2'),
Bounds3D.span,
payload_plus,
predicate=and_pred(Bounds3D.X(overlaps()), Bounds3D.Y(overlaps())))
# target3 is the same as is2_coalesced_overlapping_bboxes
target3 = IntervalSet([
Interval(Bounds3D(1, 10, 0.2, 0.8, 0.2, 0.6), 2),
Interval(Bounds3D(9, 11, 0.16, 0.17, 0.3, 0.5), 1),
Interval(Bounds3D(13, 19, 0.5, 1, 0, 0.5), 2),
])
print('is1:')
print(is1)
print()
print('is1 coalesced:')
print(is1_coalesced)
print()
print('is1_coalesced_larger_epsilon:')
print(is1_coalesced_larger_epsilon)
print()
print('is2:')
print(is2)
print()
print('is2 coalesced with overlapping bounding boxes:')
print(is2_coalesced_overlapping_bboxes)
def detect_commercials(black_frames, captions, whole_video, params=None):
"""
---
Arguments:
captions (IntervalSet):
black_frames (IntervalSet):
whole_video (IntervalSet):
params (dict):
Returns:
IntervalSet of commercials (in seconds).
"""
black_windows = black_frames.coalesce(('t1', 't2'),
Bounds3D.span,
epsilon=BLACK_FRAME_COALESCE_EPSILON)
caption_intervals = captions.filter(lambda x: '{' not in x['payload'])
arrow_intervals = caption_intervals.filter(lambda x: '>>' in x['payload'])
arrow_announcer_intervals = caption_intervals.filter(
lambda x: '>> Announcer:' in x['payload'])
arrow_having_intervals = caption_intervals.filter(
lambda x: '>> HAVING' in x['payload'])
caption_intervals = caption_intervals.coalesce(
('t1', 't2'), Bounds3D.span, epsilon=CAPTIONS_COALESCE_EPSILON)
reliable_captions = caption_intervals.filter_size(
min_size=RELIABLE_TEXT_DURATION)
arrow_intervals = arrow_intervals.minus(arrow_announcer_intervals).minus(
arrow_having_intervals).filter_against(reliable_captions,
predicate=overlaps())
# Find commercial blocks
all_blocks = whole_video.minus(black_windows)
non_commercial_blocks = all_blocks.filter_against(arrow_intervals,
predicate=overlaps())
commercial_blocks = whole_video.minus(
non_commercial_blocks.union(black_windows))
def fold_fn(stack, interval):
if interval['t2'] - interval['t1'] > MAX_COMMERCIAL_TIME:
interval = Interval(
Bounds3D(interval['t1'], interval['t1'] + MAX_COMMERCIAL_TIME))
if len(stack) == 0:
stack.append(interval)
else:
last = stack.pop()
if or_pred(overlaps(),
after(max_dist=COMMERCIAL_FOLD_EPSILON))(interval,
last):
if last['bounds'].span(interval['bounds']).size() \
> MAX_COMMERCIAL_TIME:
stack.append(
Interval(
Bounds3D(last['t1'],
last['t1'] + MAX_COMMERCIAL_TIME)))
else:
stack.append(
Interval(last['bounds'].span(interval['bounds'])))
else:
stack.append(last)
stack.append(interval)
return stack
commercials = commercial_blocks.fold_to_set(
fold_fn, init=[]).filter_size(min_size=MIN_COMMERCIAL_TIME)
commercials_orig = commercials
def is_lower_text(obj):
alpha = [c for c in obj['payload'] if c.isalpha()]
if not alpha:
return False
lower = [c for c in alpha if c.islower()]
return len(lower) / len(alpha) > MIN_LOWERTEXT
lowercase_intervals = captions.filter(is_lower_text).coalesce(
('t1', 't2'),
Bounds3D.span,
payload_merge_op=lambda a, b: a + ' ' + b,
epsilon=LOWERCASE_COALESCE_EPSILON).filter_size(
min_size=MIN_LOWERWINDOW)
commercials = commercials.union(lowercase_intervals)
blank_intervals = whole_video.minus(caption_intervals).filter_size(
min_size=MIN_BLANKWINDOW, max_size=MAX_BLANKWINDOW).minus(
whole_video.map(lambda x: Interval(Bounds3D(x['t2'] - 60, x['t2']))
)).filter_size(min_size=MIN_BLANKWINDOW)
commercials = commercials.union(blank_intervals)
commercials = commercials.coalesce(
('t1', 't2'), Bounds3D.span, epsilon=MAX_MERGE_GAP).filter_size(
max_size=MAX_COMMERCIAL_TIME).union(commercials_orig).union(
lowercase_intervals).union(blank_intervals).coalesce(
('t1', 't2'), Bounds3D.span)
return commercials