def interview_with_person_x(): from query.models import LabeledCommercial, FaceIdentity from rekall.video_interval_collection import VideoIntervalCollection from rekall.temporal_predicates import before, after, overlaps from rekall.logical_predicates import or_pred from esper.rekall import intrvllists_to_result # Get list of sandbox video IDs sandbox_videos = [ row.video_id for row in LabeledCommercial.objects.distinct('video_id') ] guest_name = "bernie sanders" # Load hosts and instances of guest from SQL identities = FaceIdentity.objects.filter(face__shot__video_id__in=sandbox_videos) hosts_qs = identities.filter(face__is_host=True) guest_qs = identities.filter(identity__name=guest_name).filter(probability__gt=0.7) # Put bounding boxes in SQL hosts = VideoIntervalCollection.from_django_qs( hosts_qs.annotate(video_id=F("face__shot__video_id"), min_frame=F("face__shot__min_frame"), max_frame=F("face__shot__max_frame")) ) guest = VideoIntervalCollection.from_django_qs( guest_qs.annotate(video_id=F("face__shot__video_id"), min_frame=F("face__shot__min_frame"), max_frame=F("face__shot__max_frame")) ) # Get all shots where the guest and a host are on screen together guest_with_host = guest.overlaps(hosts).coalesce() # This temporal predicate defines A overlaps with B, or A before by less than 10 frames, # or A after B by less than 10 frames overlaps_before_or_after_pred = or_pred( or_pred(overlaps(), before(max_dist=10), arity=2), after(max_dist=10), arity=2) # This code finds sequences of: # guest with host overlaps/before/after host OR # guest with host overlaps/before/after guest interview_candidates = guest_with_host \ .merge(hosts, predicate=overlaps_before_or_after_pred) \ .set_union(guest_with_host.merge( guest, predicate=overlaps_before_or_after_pred)) \ .coalesce() # Sequences may be interrupted by shots where the guest or host don't # appear, so dilate and coalesce to merge neighboring segments interviews = interview_candidates \ .dilate(600) \ .coalesce() \ .dilate(-600) \ .filter_length(min_length=1350) # Return intervals return intrvllists_to_result(interviews.get_allintervals())
def fold_fn(stack, interval): if interval.length() > MAX_COMMERCIAL_TIME: interval = Interval(interval.start, interval.start + MAX_COMMERCIAL_TIME, interval.payload) if len(stack) == 0: stack.append(interval) else: last = stack.pop() if or_pred(overlaps(), after(max_dist=5), arity=2)(interval, last): if last.merge(interval).length() > MAX_COMMERCIAL_TIME: stack.append(Interval( last.start, last.start + MAX_COMMERCIAL_TIME, last.payload)) else: stack.append(last.merge(interval)) else: stack.append(last) stack.append(interval) return stack
def detect_commercial_rekall(video, transcript_path, blackframe_list=None, histogram=None, verbose=True): """ API for detecting commercial blocks from TV news video using rekall @video: django query set @transcript_path: transcript_path @blackframe_list: list of black frames index @histogram: list of histogram 16x3 bin for each frame, not used if blackframe_list is provided Return: commercial_list (list of tuple((start_fid, start_sec), (end_fid, end_sec)), None if failed) """ transcript = load_transcript(transcript_path) if blackframe_list is None: blackframe_intervallist = get_blackframe_list(histogram) else: blackframe_intervallist = IntervalList([ (fid2second(fid, video.fps), fid2second(fid + 1, video.fps), 0) for fid in blackframe_list ]) black_windows = blackframe_intervallist \ .dilate(1. / video.fps) \ .coalesce() \ .dilate(-1. / video.fps) \ .filter_length(min_length=MIN_BLACKWINDOW * 1. / video.fps) # if verbose: # print("black window: ({})\n".format(black_windows.size())) # for idx, win in enumerate(black_windows.get_intervals()): # print(idx, win) # get all instances of >>, Announcer:, and >> Announcer: in transcript arrow_text = get_text_intervals(">>", transcript) announcer_text = get_text_intervals("Announcer:", transcript) arrow_announcer_text = get_text_intervals(">> Announcer:", transcript) # if verbose: # print('arrow_text', arrow_text) # print('announcer_text', announcer_text) # print('arrow_announcer_text', arrow_announcer_text) # get an interval for the whole video whole_video = IntervalList([(0., video.num_frames / video.fps, 0)]) # whole video minus black windows to get segments in between black windows # then filter out anything that overlaps with ">>" as long as it's not # ">> Announcer:" # then coalesce, as long as it doesn't get too long def fold_fn(stack, interval): if len(stack) == 0: stack.append(interval) else: last = stack.pop() if or_pred(overlaps(), after(max_dist=1), arity=2)(interval, last): if last.merge(interval).length() > MAX_COMMERCIAL_TIME: if last.length() > MAX_COMMERCIAL_TIME: stack.append( Interval(last.start, last.start + MAX_COMMERCIAL_TIME, last.payload)) else: stack.append(last) stack.append(interval) else: stack.append(last.merge(interval)) else: stack.append(last) stack.append(interval) return stack all_blocks = whole_video.minus(black_windows) non_commercial_blocks = all_blocks.filter_against( arrow_text.minus(arrow_announcer_text), predicate=overlaps()) commercial_blocks = whole_video.minus(non_commercial_blocks) if verbose: print("commercial blocks candidates: ({})\n".format( commercial_blocks.size())) for idx, win in enumerate(commercial_blocks.get_intervals()): print(idx, win) commercials = commercial_blocks \ .fold_list(fold_fn, []) \ .filter_length(min_length = MIN_COMMERCIAL_TIME) # commercials = whole_video \ # .minus(black_windows) \ # .filter_against( # arrow_text.filter_against(arrow_announcer_text, # predicate=not_pred(overlaps(), arity=2)), # predicate=not_pred(overlaps(), arity=2) # ) \ # .set_union(black_windows) \ # .fold_list(fold_fn, []) \ # .filter_length(min_length = MIN_COMMERCIAL_TIME) if verbose: print("commercials from blackwindow:\n", commercials) # add in lowercase intervals lowercase_intervals = get_lowercase_intervals(transcript) if verbose: print("lowercase intervals:\n", lowercase_intervals) commercials = commercials \ .set_union(lowercase_intervals) \ .dilate(MIN_COMMERCIAL_GAP / 2) \ .coalesce() \ .dilate(MIN_COMMERCIAL_GAP / 2) if verbose: print("commercials merge with lowercase:\n", commercials) # if verbose: # print(whole_video) # print(IntervalList([ # (start_sec - TRANSCRIPT_DELAY, end_sec - TRANSCRIPT_DELAY, 0) # for text, start_sec, end_sec in transcript # ]).coalesce().size()) # get blank intervals blank_intervals = whole_video.minus( IntervalList([ (start_sec - TRANSCRIPT_DELAY, end_sec - TRANSCRIPT_DELAY, 0) for text, start_sec, end_sec in transcript ]).coalesce()).coalesce().filter_length(min_length=MIN_BLANKWINDOW, max_length=MAX_BLANKWINDOW) if verbose: print("blank intervals:\n", blank_intervals) # add in blank intervals, but only if adding in the new intervals doesn't # get too long commercials = commercials.merge(blank_intervals, predicate=or_pred(before(max_dist=MAX_MERGE_GAP), after(max_dist=MAX_MERGE_GAP), arity=2), working_window=MAX_MERGE_GAP ) \ .filter_length(max_length=MAX_MERGE_DURATION) \ .set_union(commercials) \ .dilate(MIN_COMMERCIAL_GAP / 2) \ .coalesce() \ .dilate(MIN_COMMERCIAL_GAP / 2) if verbose: print("commercials merge with lowercase:\n", commercials) # post-process commercials to get rid of gaps, small commercials, and # islated blocks small_gaps = whole_video \ .minus(commercials) \ .filter_length(max_length = MAX_COMMERCIAL_GAP) \ .filter_against( arrow_text.filter_against( announcer_text, predicate=not_pred(overlaps()), working_window=1.0 ), predicate=not_pred(overlaps()), working_window=1.0) # merge with small gaps, but only if that doesn't make things too long commercials = commercials \ .set_union(small_gaps.dilate(0.1)) \ .coalesce() \ .filter_length(max_length=MAX_COMMERCIAL_TIME) \ .set_union(commercials) \ .coalesce() # get isolated commercials not_isolated_commercials = commercials.filter_against( commercials, predicate=or_pred(before(max_dist=MAX_COMMERCIAL_TIME), after(max_dist=MAX_COMMERCIAL_TIME), arity=2), working_window=MAX_COMMERCIAL_TIME) isolated_commercials = commercials.minus(not_isolated_commercials) commercials_to_delete = isolated_commercials \ .filter_length(max_length=MIN_COMMERCIAL_TIME_FINAL) \ .set_union(isolated_commercials \ .filter_against(blank_intervals, predicate=equal()) \ .filter_length(max_length=MAX_ISOLATED_BLANK_TIME)) commercials = commercials.minus(commercials_to_delete) return commercials
def shot_reverse_shot(): from query.models import Face from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser from rekall.merge_ops import payload_plus from esper.rekall import intrvllists_to_result_bbox from rekall.payload_predicates import payload_satisfies from rekall.list_predicates import length_at_most from rekall.logical_predicates import and_pred, or_pred from rekall.spatial_predicates import scene_graph, make_region from rekall.temporal_predicates import before, after from rekall.bbox_predicates import height_at_least from esper.rekall import intrvllists_to_result, intrvllists_to_result_with_objects, add_intrvllists_to_result # If True, visualize results in a timeline TIMELINE_OUTPUT = False RIGHT_HALF_MIN_X = 0.45 LEFT_HALF_MAX_X = 0.55 MIN_FACE_HEIGHT = 0.4 MAX_FACES_ON_SCREEN = 2 # faces are sampled every 12 frames SAMPLING_RATE = 12 ONE_SECOND = 24 FOUR_SECONDS = 96 TEN_SECONDS = 240 # Annotate face rows with start and end frames and the video ID faces = Face.objects.annotate(min_frame=F('frame__number'), max_frame=F('frame__number'), video_id=F('frame__video_id')) right_half = make_region(RIGHT_HALF_MIN_X, 0.0, 1.0, 1.0) left_half = make_region(0.0, 0.0, LEFT_HALF_MAX_X, 1.0) graph = { 'nodes': [{ 'name': 'face', 'predicates': [height_at_least(MIN_FACE_HEIGHT)] }], 'edges': [] } vids = VideoIntervalCollection.from_django_qs( faces, with_payload=in_array( bbox_payload_parser( VideoIntervalCollection.django_accessor))).coalesce( payload_merge_op=payload_plus) # Get sequences where there's a face on the right half of the screen and # there are at most two faces faces_on_right = vids.filter( and_pred(payload_satisfies(length_at_most(MAX_FACES_ON_SCREEN)), payload_satisfies(scene_graph( graph, region=right_half)))).dilate(SAMPLING_RATE / 2).coalesce() # Get sequences where there's a face on the left half of the screen and # there are at most two faces faces_on_left = vids.filter( and_pred(payload_satisfies(length_at_most(MAX_FACES_ON_SCREEN)), payload_satisfies(scene_graph( graph, region=left_half)))).dilate(SAMPLING_RATE / 2).coalesce() # Sequences where faces on left up to one second before/after faces on left # Four seconds of buffer time between left-then-right/right-then-left # segments # Only keep remaining sequences that last longer than ten seconds shot_reverse_shot = faces_on_right.merge( faces_on_left, predicate=or_pred( before(max_dist=ONE_SECOND), after(max_dist=ONE_SECOND), arity=2)).dilate(FOUR_SECONDS).coalesce().dilate( -1 * FOUR_SECONDS).filter_length(min_length=TEN_SECONDS) # Post-process to display in Esper widget if TIMELINE_OUTPUT: results = intrvllists_to_result(shot_reverse_shot.get_allintervals()) add_intrvllists_to_result(results, faces_on_left.get_allintervals(), color='black') add_intrvllists_to_result(results, faces_on_right.get_allintervals(), color='green') else: results = intrvllists_to_result_with_objects( shot_reverse_shot.get_allintervals(), lambda payload, video: []) return results
def conversations_for_display(): from query.models import FaceCharacterActor, Shot from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser from rekall.merge_ops import payload_plus from rekall.payload_predicates import payload_satisfies from rekall.spatial_predicates import scene_graph from esper.rekall import intrvllists_to_result_bbox from query.models import Face from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser from rekall.merge_ops import payload_plus, merge_named_payload, payload_second from esper.rekall import intrvllists_to_result_bbox from rekall.payload_predicates import payload_satisfies from rekall.list_predicates import length_at_most from rekall.logical_predicates import and_pred, or_pred, true_pred from rekall.spatial_predicates import scene_graph, make_region from rekall.temporal_predicates import before, after, overlaps, equal from rekall.bbox_predicates import height_at_least from esper.rekall import intrvllists_to_result, intrvllists_to_result_with_objects, add_intrvllists_to_result from esper.prelude import esper_widget from rekall.interval_list import Interval, IntervalList import esper.face_embeddings as face_embeddings video_id = 15 EMBEDDING_EQUALITY_THRESHOLD = 1. ONE_FRAME = 1 faces_qs = Face.objects.annotate(min_frame=F('frame__number'), max_frame=F('frame__number'), video_id=F('frame__video_id')).filter( frame__video_id=video_id, frame__regularly_sampled=True) faces_per_frame = VideoIntervalCollection.from_django_qs( faces_qs, with_payload=in_array( merge_dict_parsers([ bbox_payload_parser(VideoIntervalCollection.django_accessor), dict_payload_parser(VideoIntervalCollection.django_accessor, {'face_id': 'id'}), ]))).coalesce(payload_merge_op=payload_plus) shots_qs = Shot.objects.filter(cinematic=True) shots = VideoIntervalCollection.from_django_qs(shots_qs) shots_with_faces = shots.merge( faces_per_frame, predicate=overlaps(), payload_merge_op=lambda shot_id, faces_in_frame: (shot_id, [faces_in_frame])).coalesce( payload_merge_op=lambda p1, p2: (p1[0], p1[1] + p2[1])) def cluster_center(face_ids): # print("About to compute mean") mean_embedding = face_embeddings.mean(face_ids) # print("About to compute dist", face_ids) dists = face_embeddings.dist(face_ids, [mean_embedding]) # print("Done computing dist") return min(zip(dists, face_ids))[1] def cluster_and_compute_centers(faces_in_frame_list, shot_id): num_people = max( len(faces_in_frame) for faces_in_frame in faces_in_frame_list) face_ids = [ face['face_id'] for faces_in_frame in faces_in_frame_list for face in faces_in_frame ] face_heights = [ face['y2'] - face['y1'] for faces_in_frame in faces_in_frame_list for face in faces_in_frame ] print(num_people) if num_people == 1: clusters = [(fid, 0) for fid in face_ids] else: clusters = face_embeddings.kmeans(face_ids, num_people) # print("Done clustering") centers = [(cluster_center([ face_id for face_id, cluster_id in clusters if cluster_id == i ]), [face_id for face_id, cluster_id in clusters if cluster_id == i], shot_id, max([ face_heights[face_ids.index(face_id)] for face_id, cluster_id in clusters if cluster_id == i ])) for i in range(num_people)] # print("Done computing the center") return centers # print("About to compute clusters") shots_with_centers = shots_with_faces.map(lambda intrvl: ( intrvl.start, intrvl.end, (intrvl.payload[0], cluster_and_compute_centers(intrvl.payload[1], intrvl.payload[0])))) # print("Clusters computed") def same_face(center1, center2): return face_embeddings.dist( [center1], target_ids=[center2])[0] < EMBEDDING_EQUALITY_THRESHOLD def cross_product_faces(intrvl1, intrvl2): payload1 = intrvl1.get_payload() payload2 = intrvl2.get_payload() payload = [] for cluster1 in payload1[1]: for cluster2 in payload2[1]: if not same_face(cluster1[0], cluster2[0]): new_payload = {'A': cluster1, 'B': cluster2} payload.append(new_payload) return [(min(intrvl1.get_start(), intrvl2.get_start()), max(intrvl1.get_end(), intrvl2.get_end()), { 'chrs': payload, 'shots': [payload1[0], payload2[0]] })] two_shots = shots_with_centers.join(shots_with_centers, predicate=after(max_dist=ONE_FRAME, min_dist=ONE_FRAME), merge_op=cross_product_faces) # print("Cross product done") def faces_equal(payload1, payload2): for face_pair1 in payload1['chrs']: for face_pair2 in payload2['chrs']: if (same_face(face_pair1['A'][0], face_pair2['A'][0]) and same_face(face_pair1['B'][0], face_pair2['B'][0])): return True if (same_face(face_pair1['A'][0], face_pair2['B'][0]) and same_face(face_pair1['B'][0], face_pair2['A'][0])): return True return False convs = two_shots.coalesce( predicate=payload_satisfies(faces_equal, arity=2), payload_merge_op=lambda payload1, payload2: { 'chrs': payload1['chrs'] + payload2['chrs'], 'shots': payload1['shots'] + payload2['shots'] }) # print("Coalesce done") adjacent_seq = convs.merge( convs, predicate=and_pred(after(max_dist=ONE_FRAME, min_dist=ONE_FRAME), payload_satisfies(faces_equal, arity=2), arity=2), payload_merge_op=lambda payload1, payload2: { 'chrs': payload1['chrs'] + payload2['chrs'], 'shots': payload1['shots'] + payload2['shots'] }, working_window=1) convs = convs.set_union(adjacent_seq) # convs = convs.coalesce(predicate=times_equal, payload_merge_op=shots_equal) # print("Two-shot adjacencies done") def filter_fn(intvl): payload = intvl.get_payload() if type(payload) is dict and 'shots' in payload: return len(set(payload['shots'])) >= 3 return False convs = convs.filter(filter_fn) convs = convs.coalesce() # print("Final filter done") # for video_id in convs.intervals.keys(): # print(video_id) # intvllist = convs.get_intervallist(video_id) # for intvl in intvllist.get_intervals(): # print(intvl.payload) # print(str(intvl.start) + ':' + str(intvl.end)) return intervallists_to_result_with_objects(convs, lambda a, b: [])