def test_filter_length(self):
intrvl1 = Interval(1., 2., 1)
intrvl2 = Interval(1.5, 3.5, 2)
intrvls1 = IntervalList([intrvl1, intrvl2])
intrvls1 = intrvls1.filter_length(min_length=1.1)
self.assertEqual(len(intrvls1.intrvls), 1)
self.assertEqual(intrvls1.intrvls[0].__repr__(),
"<Interval start:1.5 end:3.5 payload:2>")
intrvls1 = intrvls1.filter_length(max_length=1.8)
self.assertEqual(len(intrvls1.intrvls), 0)
def detect_commercial_rekall(video, transcript_path, blackframe_list=None, histogram=None, debug=True, verbose=False):
"""
API for detecting commercial blocks from TV news video using rekall
@video: django query set
@transcript_path: transcript_path
@blackframe_list: list of black frames index
@histogram: list of histogram 16x3 bin for each frame, not used if blackframe_list is provided
Return: commercial_list (list of tuple((start_fid, start_sec), (end_fid, end_sec)), None if failed)
"""
if type(video) == dict:
fps = video['fps']
video_length = video['num_frames'] / fps
else:
fps = video.fps
video_length = video.num_frames / video.fps
transcript = load_transcript(transcript_path)
if blackframe_list is None:
blackframe_intervallist = get_blackframe_list(histogram)
else:
blackframe_intervallist = IntervalList([(fid2second(fid, fps),
fid2second(fid + 1, fps),
0) for fid in blackframe_list])
# get black windows
black_windows = blackframe_intervallist \
.dilate(1. / fps) \
.coalesce() \
.dilate(-1. / fps)
# .filter_length(min_length=MIN_BLACKWINDOW * 1. / video.fps)
if verbose:
print("black window: ({})\n".format(black_windows.size()))
for idx, win in enumerate(black_windows.get_intervals()):
print(idx, win)
# get all instances of >>
arrow_intervals = get_text_intervals(">>", transcript)
arrow_announcer_intervals = get_text_intervals(">> Announcer:", transcript)
arrow_having_intervals = get_text_intervals(">> HAVING", transcript)
if verbose:
print("arrow_text: ({})\n".format(arrow_intervals.size()), arrow_intervals)
print("arrow_announcer_text: ({})\n".format(arrow_announcer_intervals.size()), arrow_announcer_intervals)
# get intervals for the whole transcript
transcript_intervals = IntervalList([
(start_sec, end_sec, 0)
for text, start_sec, end_sec in transcript
if not '{' in text
]).dilate(1) \
.coalesce() \
.dilate(-1) \
# get an interval for the whole video
whole_video = IntervalList([(0., video_length, 0)])
# whole video minus black windows to get segments in between black windows
# then filter out anything that overlaps with ">>" as long as it's not ">> Announcer:"
# then coalesce, as long as it doesn't get too long
def fold_fn(stack, interval):
if interval.length() > MAX_COMMERCIAL_TIME:
interval = Interval(interval.start, interval.start + MAX_COMMERCIAL_TIME, interval.payload)
if len(stack) == 0:
stack.append(interval)
else:
last = stack.pop()
if or_pred(overlaps(), after(max_dist=5), arity=2)(interval, last):
if last.merge(interval).length() > MAX_COMMERCIAL_TIME:
stack.append(Interval(
last.start,
last.start + MAX_COMMERCIAL_TIME,
last.payload))
else:
stack.append(last.merge(interval))
else:
stack.append(last)
stack.append(interval)
return stack
# get reliable double arrow intervals
reliable_transcripts = transcript_intervals.filter_length(min_length=RELIABLE_TEXT_DURATION)
arrow_intervals = arrow_intervals \
.minus(arrow_announcer_intervals) \
.minus(arrow_having_intervals) \
.filter_against(
reliable_transcripts,
predicate=overlaps()
)
# get non-commercial blocks by filtering out intervals overlaps with >>
all_blocks = whole_video.minus(black_windows)
non_commercial_blocks = all_blocks.filter_against(
arrow_intervals,
predicate=overlaps()
)
commercial_blocks = whole_video.minus(non_commercial_blocks.set_union(black_windows))
if verbose:
print("commercial blocks candidates: ({})\n".format(commercial_blocks.size()))
for idx, win in enumerate(commercial_blocks.get_intervals()):
print(idx, win)
commercials = commercial_blocks \
.fold_list(fold_fn, []) \
.filter_length(min_length = MIN_COMMERCIAL_TIME)
commercials_raw = copy.deepcopy(commercials)
if verbose:
print("commercials from blackwindow:\n", commercials)
# add in lowercase intervals
lowercase_intervals = get_lowercase_intervals(transcript)
if verbose:
print("lowercase intervals:\n", lowercase_intervals)
commercials = commercials.set_union(lowercase_intervals)
if verbose:
print("commercials merge with lowercase:\n", commercials)
# get blank intervals
blank_intervals = whole_video \
.minus(transcript_intervals) \
.filter_length(min_length=MIN_BLANKWINDOW, max_length=MAX_BLANKWINDOW)
# remove last one minute segment due to no aligned transcripts
blank_intervals = blank_intervals \
.minus(IntervalList([(video_length-60, video_length, 0)])) \
.filter_length(min_length=MIN_BLANKWINDOW)
if verbose:
print("blank intervals:\n", blank_intervals)
# add in blank intervals
commercials = commercials.set_union(blank_intervals)
# commercials = commercials.merge(blank_intervals,
# predicate=or_pred(before(max_dist=MAX_MERGE_GAP),
# after(max_dist=MAX_MERGE_GAP), arity=2),
# working_window=MAX_MERGE_GAP
# ) \
# .filter_length(max_length=MAX_MERGE_DURATION) \
# .set_union(commercials) \
# .dilate(MIN_COMMERCIAL_GAP / 2) \
# .coalesce() \
# .dilate(-MIN_COMMERCIAL_GAP / 2)
if verbose:
print("commercials merge with blank intervals:\n", commercials)
# merge with small gaps, but only if that doesn't make things too long
commercials = commercials \
.dilate(MAX_MERGE_GAP / 2) \
.coalesce() \
.dilate(-MAX_MERGE_GAP / 2) \
.filter_length(max_length=MAX_COMMERCIAL_TIME) \
.set_union(commercials_raw) \
.set_union(lowercase_intervals) \
.set_union(blank_intervals) \
.coalesce()
# # post-process commercials to get rid of gaps, small commercials, and
# # islated blocks
# small_gaps = whole_video \
# .minus(commercials) \
# .filter_length(max_length = MAX_COMMERCIAL_GAP) \
# .filter_against(
# arrow_text.filter_against(
# announcer_text,
# predicate=not_pred(overlaps()),
# working_window=1.0
# ), predicate=not_pred(overlaps()),
# working_window=1.0)
# # merge with small gaps, but only if that doesn't make things too long
# commercials = commercials \
# .set_union(small_gaps.dilate(0.1)) \
# .coalesce() \
# .filter_length(max_length=MAX_COMMERCIAL_TIME) \
# .set_union(commercials) \
# .coalesce()
# # get isolated commercials
# not_isolated_commercials = commercials.filter_against(commercials,
# predicate=or_pred(before(max_dist=MAX_COMMERCIAL_TIME),
# after(max_dist=MAX_COMMERCIAL_TIME), arity=2),
# working_window=MAX_COMMERCIAL_TIME)
# isolated_commercials = commercials.minus(not_isolated_commercials)
# commercials_to_delete = isolated_commercials \
# .filter_length(max_length=MIN_COMMERCIAL_TIME_FINAL) \
# .set_union(isolated_commercials \
# .filter_against(blank_intervals, predicate=equal()) \
# .filter_length(max_length=MAX_ISOLATED_BLANK_TIME))
# commercials = commercials.minus(commercials_to_delete)
if debug:
result = {'black': black_windows.dilate(2),
'arrow': arrow_intervals.dilate(2),
'commercials_raw': commercials_raw,
'lowercase': lowercase_intervals,
'blank': blank_intervals,
'commercials': commercials,
}
return result
else:
result = [(i.start, i.end) for i in commercials.get_intervals()]
return result
