def all_faces_rekall(): from query.models import Face from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser from rekall.merge_ops import payload_plus from esper.rekall import intrvllists_to_result_bbox from esper.stdlib import qs_to_result video_name = "inception" # Annotate face rows with start and end frames and the video ID faces = Face.objects.filter(frame__video__name=video_name).annotate( min_frame=F('frame__number'), max_frame=F('frame__number'), video_id=F('frame__video_id')) # Materialize all the faces and load them into rekall with bounding box payloads # Then coalesce them so that all faces in the same frame are in the same interval # NOTE that this is slow right now since we're loading all faces! vids = VideoIntervalCollection.from_django_qs( faces, with_payload=in_array( bbox_payload_parser( VideoIntervalCollection.django_accessor))).coalesce( payload_merge_op=payload_plus) # Post-process to display in Esper widget return intrvllists_to_result_bbox(vids.get_allintervals(), limit=100, stride=100)
def faces_with_gender(): from query.models import FaceGender from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser from rekall.merge_ops import payload_plus from esper.rekall import intrvllists_to_result_bbox VIDEO_NAME_CONTAINS = "harry potter" # Annotate face rows with start and end frames and the video ID faces_with_gender = FaceGender.objects.annotate( min_frame=F('face__frame__number'), max_frame=F('face__frame__number'), video_id=F('face__frame__video_id'), bbox_x1=F('face__bbox_x1'), bbox_y1=F('face__bbox_y1'), bbox_x2=F('face__bbox_x2'), bbox_y2=F('face__bbox_y2'), gender_name=F('gender__name')).filter( face__frame__video__name__contains=VIDEO_NAME_CONTAINS) faces = VideoIntervalCollection.from_django_qs( faces_with_gender, with_payload=in_array( merge_dict_parsers([ bbox_payload_parser(VideoIntervalCollection.django_accessor), dict_payload_parser(VideoIntervalCollection.django_accessor, {'gender': 'gender_name'}) ]))).coalesce(payload_merge_op=payload_plus) return intrvllists_to_result_bbox(faces.get_allintervals(), limit=100, stride=1000)
def three_people(): from query.models import Face from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser from rekall.merge_ops import payload_plus from esper.rekall import intrvllists_to_result_bbox from rekall.payload_predicates import payload_satisfies from rekall.spatial_predicates import scene_graph from rekall.bbox_predicates import height_at_least, left_of, same_value MIN_FACE_HEIGHT = 0.3 EPSILON = 0.05 # Annotate face rows with start and end frames and the video ID faces = Face.objects.annotate(min_frame=F('frame__number'), max_frame=F('frame__number'), video_id=F('frame__video_id')) # Materialize all the faces and load them into rekall with bounding box payloads # Then coalesce them so that all faces in the same frame are in the same interval # NOTE that this is slow right now since we're loading all faces! face_lists = VideoIntervalCollection.from_django_qs( faces, with_payload=in_array( bbox_payload_parser( VideoIntervalCollection.django_accessor))).coalesce( payload_merge_op=payload_plus) three_people_scene_graph = { 'nodes': [{ 'name': 'face1', 'predicates': [height_at_least(MIN_FACE_HEIGHT)] }, { 'name': 'face2', 'predicates': [height_at_least(MIN_FACE_HEIGHT)] }, { 'name': 'face3', 'predicates': [height_at_least(MIN_FACE_HEIGHT)] }], 'edges': [{ 'start': 'face1', 'end': 'face2', 'predicates': [left_of(), same_value('y1', epsilon=EPSILON)] }, { 'start': 'face2', 'end': 'face3', 'predicates': [left_of(), same_value('y1', epsilon=EPSILON)] }] } three_people = face_lists.filter( payload_satisfies(scene_graph(three_people_scene_graph, exact=True))) # Post-process to display in Esper widget return intrvllists_to_result_bbox(three_people.get_allintervals(), limit=100, stride=100)
def man_woman_up_close(): from query.models import FaceGender from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser from rekall.merge_ops import payload_plus from esper.rekall import intrvllists_to_result_bbox from rekall.payload_predicates import payload_satisfies from rekall.spatial_predicates import scene_graph from rekall.bbox_predicates import height_at_least MIN_FACE_CONFIDENCE = 0.95 MIN_GENDER_CONFIDENCE = 0.95 MIN_FACE_HEIGHT = 0.6 # Annotate face rows with start and end frames and the video ID faces_with_gender= FaceGender.objects.filter(face__frame__video__name=video_name).annotate( min_frame=F('face__frame__number'), max_frame=F('face__frame__number'), video_id=F('face__frame__video_id'), bbox_x1=F('face__bbox_x1'), bbox_y1=F('face__bbox_y1'), bbox_x2=F('face__bbox_x2'), bbox_y2=F('face__bbox_y2'), gender_name=F('gender__name'), face_probability=F('face__probability')) faces = VideoIntervalCollection.from_django_qs( faces_with_gender, with_payload=in_array(merge_dict_parsers([ bbox_payload_parser(VideoIntervalCollection.django_accessor), dict_payload_parser(VideoIntervalCollection.django_accessor, { 'gender': 'gender_name' }), dict_payload_parser(VideoIntervalCollection.django_accessor, { 'gender_probability': 'probability' }), dict_payload_parser(VideoIntervalCollection.django_accessor, { 'face_probability': 'face_probability' }) ])) ).coalesce(payload_merge_op=payload_plus) graph = { 'nodes': [ { 'name': 'face_male', 'predicates': [ height_at_least(MIN_FACE_HEIGHT), lambda payload: payload['gender'] is 'M', lambda payload: payload['face_probability'] > MIN_FACE_CONFIDENCE, lambda payload: payload['gender_probability'] > MIN_GENDER_CONFIDENCE ] }, { 'name': 'face_female', 'predicates': [ height_at_least(MIN_FACE_HEIGHT), lambda payload: payload['gender'] is 'F', lambda payload: payload['face_probability'] > MIN_FACE_CONFIDENCE, lambda payload: payload['gender_probability'] > MIN_GENDER_CONFIDENCE ] }, ], 'edges': [] } mf_up_close = faces.filter(payload_satisfies( scene_graph(graph, exact=True))) return intrvllists_to_result_bbox(mf_up_close.get_allintervals(), limit=100, stride=100)
def panels_rekall(): from query.models import LabeledCommercial, Face from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser from rekall.merge_ops import payload_plus from rekall.bbox_predicates import height_at_least, same_value, left_of from rekall.spatial_predicates import scene_graph from rekall.payload_predicates import payload_satisfies from esper.rekall import intrvllists_to_result_bbox # Get list of sandbox video IDs sandbox_videos = [ row.video_id for row in LabeledCommercial.objects.distinct('video_id') ] faces_qs = Face.objects.filter(shot__video_id__in=sandbox_videos).annotate( video_id=F("shot__video_id"), min_frame=F("shot__min_frame"), max_frame=F("shot__max_frame") ) # One interval for each face faces = VideoIntervalCollection.from_django_qs( faces_qs, with_payload=in_array( bbox_payload_parser( VideoIntervalCollection.django_accessor))) # Merge shots faces = faces.coalesce(payload_merge_op=payload_plus) # Define a scene graph for things that look like panels three_faces_scene_graph = { 'nodes': [ { 'name': 'face1', 'predicates': [ height_at_least(0.3) ] }, { 'name': 'face2', 'predicates': [ height_at_least(0.3) ] }, { 'name': 'face3', 'predicates': [ height_at_least(0.3) ] } ], 'edges': [ { 'start': 'face1', 'end': 'face2', 'predicates': [ same_value('y1', epsilon=0.05), left_of() ] }, { 'start': 'face2', 'end': 'face3', 'predicates': [ same_value('y1', epsilon=0.05), left_of() ] }, ] } panels = faces.filter(payload_satisfies( scene_graph(three_faces_scene_graph, exact=True) )) return intrvllists_to_result_bbox(panels.get_allintervals())
def frames_with_character_x(): from query.models import FaceCharacterActor from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser from rekall.merge_ops import payload_plus from rekall.payload_predicates import payload_satisfies from rekall.spatial_predicates import scene_graph from esper.rekall import intrvllists_to_result_bbox character_name = "harry potter" # Annotate face rows with start and end frames and the video ID faces_with_character_actor_qs = FaceCharacterActor.objects.annotate( min_frame=F('face__frame__number'), max_frame=F('face__frame__number'), video_id=F('face__frame__video_id'), bbox_x1=F('face__bbox_x1'), bbox_y1=F('face__bbox_y1'), bbox_x2=F('face__bbox_x2'), bbox_y2=F('face__bbox_y2'), character_name=F('characteractor__character__name')) faces_with_identity = VideoIntervalCollection.from_django_qs( faces_with_character_actor_qs, with_payload=in_array( merge_dict_parsers([ bbox_payload_parser(VideoIntervalCollection.django_accessor), dict_payload_parser(VideoIntervalCollection.django_accessor, {'character': 'character_name'}), ]))).coalesce(payload_merge_op=payload_plus) faces_with_actor = faces_with_identity.filter( payload_satisfies( scene_graph({ 'nodes': [{ 'name': 'face1', 'predicates': [lambda f: f['character'] == character_name] }], 'edges': [] }))) return intrvllists_to_result_bbox(faces_with_actor.get_allintervals(), limit=100, stride=1000)
def get_numface_intrvlcol(relevant_shots, num_face=1): faces = Face.objects.filter(shot__in=list(relevant_shots)) \ .annotate(video_id=F('shot__video_id')) \ .annotate(min_frame=F('shot__min_frame')) \ .annotate(max_frame=F('shot__max_frame')) # Materialize all the faces and load them into rekall with bounding box payloads # Then coalesce them so that all faces in the same frame are in the same interval # NOTE that this is slow right now since we're loading all faces! numface_intrvlcol = VideoIntervalCollection.from_django_qs( faces, with_payload=in_array( bbox_payload_parser(VideoIntervalCollection.django_accessor)) ).coalesce(payload_merge_op=payload_plus).filter(payload_satisfies(length_exactly(num_face))) num_intrvl = 0 for _, intrvllist in numface_intrvlcol.get_allintervals().items(): num_intrvl += intrvllist.size() print("Get {} relevant {} face intervals".format(num_intrvl, num_face)) return numface_intrvlcol
def shot_reverse_shot_complex(): from query.models import Face, Shot from rekall.temporal_predicates import overlaps from rekall.merge_ops import payload_second, payload_plus from rekall.video_interval_collection import VideoIntervalCollection from rekall.interval_list import Interval, IntervalList from rekall.parsers import in_array, bbox_payload_parser from rekall.payload_predicates import payload_satisfies from rekall.list_predicates import length_at_most from rekall.logical_predicates import and_pred from rekall.spatial_predicates import scene_graph, make_region from rekall.temporal_predicates import before, after from rekall.bbox_predicates import height_at_least from esper.rekall import intrvllists_to_result_with_objects VIDEO_NAME = 'godfather part iii' MAX_FACE_MOVEMENT = 0.15 MIN_FACE_HEIGHT = 0.2 MAX_FACES_ON_SCREEN = 4 RIGHT_HALF_MIN_X = 0.33 LEFT_HALF_MAX_X = 0.66 SHOTS_LABELER_ID = 64 # faces are sampled every 12 frames SAMPLING_RATE = 12 # Annotate face rows with start and end frames and the video ID faces = Face.objects.annotate(min_frame=F('frame__number'), max_frame=F('frame__number'), video_id=F('frame__video_id')).filter( frame__video__name__contains=VIDEO_NAME) shots = VideoIntervalCollection.from_django_qs(Shot.objects.filter( video__name__contains=VIDEO_NAME, labeler_id=SHOTS_LABELER_ID), with_payload=lambda obj: []) # vids are all faces for each frame vids = VideoIntervalCollection.from_django_qs( faces.filter(probability__gte=0.99), with_payload=in_array( bbox_payload_parser( VideoIntervalCollection.django_accessor))).coalesce( payload_merge_op=payload_plus) right_half = make_region(RIGHT_HALF_MIN_X, 0.0, 1.0, 1.0) left_half = make_region(0.0, 0.0, LEFT_HALF_MAX_X, 1.0) graph = { 'nodes': [{ 'name': 'face', 'predicates': [height_at_least(MIN_FACE_HEIGHT)] }], 'edges': [] } faces_on_right = vids.filter( and_pred(payload_satisfies(length_at_most(MAX_FACES_ON_SCREEN)), payload_satisfies(scene_graph(graph, region=right_half)))) faces_on_left = vids.filter( and_pred(payload_satisfies(length_at_most(MAX_FACES_ON_SCREEN)), payload_satisfies(scene_graph(graph, region=left_half)))) def wrap_list(intvl): intvl.payload = [intvl.payload] return intvl def get_height(box): return box['y2'] - box['y1'] def get_center(box): return ((box['x1'] + box['x2']) / 2, (box['y1'] + box['y2']) / 2) def get_distance(pt1, pt2): return np.sqrt((pt1[0] - pt2[0])**2 + (pt1[1] - pt2[1])**2) def find_highest_box(boxes): if len(boxes) == 0: return None result = boxes[0] best = get_height(result) for i in range(1, len(boxes)): h = get_height(boxes[i]) if h > best: best = h result = boxes[i] return result def take_highest_in_frame(intvl): result = [] for faces_in_frame in intvl.payload: largest = find_highest_box(faces_in_frame) if largest is not None: result.append(largest) intvl.payload = result return intvl # Check if displacement of box center between frames are within `dist` def inter_frame_movement_less_than(dist): def check(boxes): for b1, b2 in zip(boxes, boxes[1:]): if get_distance(get_center(b1), get_center(b2)) > dist: return False return True return check # Payload is a list, each element is a list of faces for a frame shots_with_face_on_right = shots.merge( faces_on_right, predicate=overlaps(), payload_merge_op=payload_second).map(wrap_list).coalesce( payload_merge_op=payload_plus).map(take_highest_in_frame).filter( payload_satisfies( inter_frame_movement_less_than(MAX_FACE_MOVEMENT))) shots_with_face_on_left = shots.merge( faces_on_left, predicate=overlaps(), payload_merge_op=payload_second).map(wrap_list).coalesce( payload_merge_op=payload_plus).map(take_highest_in_frame).filter( payload_satisfies( inter_frame_movement_less_than(MAX_FACE_MOVEMENT))) # Right-Left-Right sequences shot_reverse_shot_1 = shots_with_face_on_right.merge( shots_with_face_on_left, predicate=before(max_dist=1)).merge(shots_with_face_on_right, predicate=before(max_dist=1)) # Left-Right-Left sequences shot_reverse_shot_2 = shots_with_face_on_left.merge( shots_with_face_on_right, predicate=before(max_dist=1)).merge(shots_with_face_on_left, predicate=before(max_dist=1)) shot_reverse_shot = shot_reverse_shot_1.set_union( shot_reverse_shot_2).coalesce() result = intrvllists_to_result_with_objects( shot_reverse_shot.get_allintervals(), payload_to_objs=lambda p, v: []) return result
def shot_reverse_shot(): from query.models import Face from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser from rekall.merge_ops import payload_plus from esper.rekall import intrvllists_to_result_bbox from rekall.payload_predicates import payload_satisfies from rekall.list_predicates import length_at_most from rekall.logical_predicates import and_pred, or_pred from rekall.spatial_predicates import scene_graph, make_region from rekall.temporal_predicates import before, after from rekall.bbox_predicates import height_at_least from esper.rekall import intrvllists_to_result, intrvllists_to_result_with_objects, add_intrvllists_to_result # If True, visualize results in a timeline TIMELINE_OUTPUT = False RIGHT_HALF_MIN_X = 0.45 LEFT_HALF_MAX_X = 0.55 MIN_FACE_HEIGHT = 0.4 MAX_FACES_ON_SCREEN = 2 # faces are sampled every 12 frames SAMPLING_RATE = 12 ONE_SECOND = 24 FOUR_SECONDS = 96 TEN_SECONDS = 240 # Annotate face rows with start and end frames and the video ID faces = Face.objects.annotate(min_frame=F('frame__number'), max_frame=F('frame__number'), video_id=F('frame__video_id')) right_half = make_region(RIGHT_HALF_MIN_X, 0.0, 1.0, 1.0) left_half = make_region(0.0, 0.0, LEFT_HALF_MAX_X, 1.0) graph = { 'nodes': [{ 'name': 'face', 'predicates': [height_at_least(MIN_FACE_HEIGHT)] }], 'edges': [] } vids = VideoIntervalCollection.from_django_qs( faces, with_payload=in_array( bbox_payload_parser( VideoIntervalCollection.django_accessor))).coalesce( payload_merge_op=payload_plus) # Get sequences where there's a face on the right half of the screen and # there are at most two faces faces_on_right = vids.filter( and_pred(payload_satisfies(length_at_most(MAX_FACES_ON_SCREEN)), payload_satisfies(scene_graph( graph, region=right_half)))).dilate(SAMPLING_RATE / 2).coalesce() # Get sequences where there's a face on the left half of the screen and # there are at most two faces faces_on_left = vids.filter( and_pred(payload_satisfies(length_at_most(MAX_FACES_ON_SCREEN)), payload_satisfies(scene_graph( graph, region=left_half)))).dilate(SAMPLING_RATE / 2).coalesce() # Sequences where faces on left up to one second before/after faces on left # Four seconds of buffer time between left-then-right/right-then-left # segments # Only keep remaining sequences that last longer than ten seconds shot_reverse_shot = faces_on_right.merge( faces_on_left, predicate=or_pred( before(max_dist=ONE_SECOND), after(max_dist=ONE_SECOND), arity=2)).dilate(FOUR_SECONDS).coalesce().dilate( -1 * FOUR_SECONDS).filter_length(min_length=TEN_SECONDS) # Post-process to display in Esper widget if TIMELINE_OUTPUT: results = intrvllists_to_result(shot_reverse_shot.get_allintervals()) add_intrvllists_to_result(results, faces_on_left.get_allintervals(), color='black') add_intrvllists_to_result(results, faces_on_right.get_allintervals(), color='green') else: results = intrvllists_to_result_with_objects( shot_reverse_shot.get_allintervals(), lambda payload, video: []) return results
def kissing(): # Takes 7min to run! from query.models import Face, Shot from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser from rekall.merge_ops import payload_plus from rekall.payload_predicates import payload_satisfies from rekall.spatial_predicates import scene_graph from rekall.temporal_predicates import overlaps from rekall.face_landmark_predicates import looking_left, looking_right from rekall.bbox_predicates import height_at_least, same_height import esper.face_landmarks_wrapper as flw from esper.captions import get_all_segments from esper.rekall import intrvllists_to_result_with_objects, bbox_to_result_object from esper.stdlib import face_landmarks_to_dict MAX_MOUTH_DIFF = 0.12 MIN_FACE_CONFIDENCE = 0.8 MIN_FACE_HEIGHT = 0.4 MAX_FACE_HEIGHT_DIFF = 0.1 MIN_FACE_OVERLAP_X = 0.05 MIN_FACE_OVERLAP_Y = 0.2 MAX_FACE_OVERLAP_X_FRACTION = 0.7 MIN_FACE_ANGLE = 0.1 def map_payload(func): def map_fn(intvl): intvl.payload = func(intvl.payload) return intvl return map_fn def get_landmarks(faces): ids = [face['id'] for face in faces] landmarks = flw.get(Face.objects.filter(id__in=ids)) for face, landmark in zip(faces, landmarks): face['landmarks'] = landmark return faces # Annotate face rows with start and end frames and the video ID faces_qs = Face.objects.filter( probability__gte=MIN_FACE_CONFIDENCE).annotate( min_frame=F('frame__number'), max_frame=F('frame__number'), height=F('bbox_y2') - F('bbox_y1'), video_id=F('frame__video_id')).filter(height__gte=MIN_FACE_HEIGHT) faces = VideoIntervalCollection.from_django_qs( faces_qs, with_payload=in_array( merge_dict_parsers([ bbox_payload_parser(VideoIntervalCollection.django_accessor), dict_payload_parser(VideoIntervalCollection.django_accessor, {'id': 'id'}) ]))).coalesce(payload_merge_op=payload_plus) graph = { 'nodes': [ { 'name': 'face_left', 'predicates': [] }, { 'name': 'face_right', 'predicates': [] }, ], 'edges': [ { 'start': 'face_left', 'end': 'face_right', 'predicates': [ lambda f1, f2: f1['x2'] < f2['x2'] and f1['x1'] < f2[ 'x1'], # Left face on the left lambda f1, f2: f1['x2'] - f2['x1'] > MIN_FACE_OVERLAP_X, # Faces overlap lambda f1, f2: min(f1['y2'], f2['y2']) - max( f1['y1'], f1['y1']) > MIN_FACE_OVERLAP_Y, lambda f1, f2: f1['y2'] > f2['y1'] and f1['y1'] < f2[ 'y2'], # No face is entirely above another same_height(MAX_FACE_HEIGHT_DIFF), lambda f1, f2: (f1['x2'] - f2['x1']) / max(f1['x2'] - f1['x1'], f2[ 'x2'] - f2['x1']) < MAX_FACE_OVERLAP_X_FRACTION ] }, ] } def mouths_are_close(lm1, lm2): select_outer = [2, 3, 4, 8, 9, 10] select_inner = [1, 2, 3, 5, 6, 7] mouth1 = np.concatenate( (lm1.outer_lips()[select_outer], lm1.inner_lips()[select_inner])) mouth2 = np.concatenate( (lm2.outer_lips()[select_outer], lm2.inner_lips()[select_inner])) mean1 = np.mean(mouth1, axis=0) mean2 = np.mean(mouth2, axis=0) return np.linalg.norm(mean1 - mean2) <= MAX_MOUTH_DIFF # Face is profile if both eyes are on the same side of the nose bridge horizontally. def is_left_profile(f): lm = f['landmarks'] nose_x = min(lm.nose_bridge()[:, 0]) left = np.all(lm.left_eye()[:, 0] >= nose_x) right = np.all(lm.right_eye()[:, 0] >= nose_x) return left and right def is_right_profile(f): lm = f['landmarks'] nose_x = max(lm.nose_bridge()[:, 0]) left = np.all(lm.left_eye()[:, 0] <= nose_x) right = np.all(lm.right_eye()[:, 0] <= nose_x) return left and right # Line is ax+by+c=0 def project_point_to_line(pt, a, b, c): x0, y0 = pt[0], pt[1] d = a * a + b * b x = (b * (b * x0 - a * y0) - a * c) / d y = (a * (-b * x0 + a * y0) - b * c) / d return np.array([x, y]) # Positive if facing right def signed_face_angle(lm): center_line_indices = [27, 28, 32, 33, 34, 51, 62, 66, 57] data = lm.landmarks[center_line_indices] fit = np.polyfit(data[:, 0], data[:, 1], 1) # y = ax+b a, b = fit[0], fit[1] A = project_point_to_line(lm.landmarks[center_line_indices[0]], a, -1, b) B = project_point_to_line(lm.landmarks[center_line_indices[-1]], a, -1, b) AB = B - A AB = AB / np.linalg.norm(AB) C = np.mean(lm.nose_bridge()[2:4], axis=0) AC = C - A AC = AC / np.linalg.norm(AC) return np.cross(AB, AC) graph2 = { 'nodes': [ { 'name': 'left', 'predicates': [ lambda f: signed_face_angle(f['landmarks']) > MIN_FACE_ANGLE # is_right_profile ] }, { 'name': 'right', 'predicates': [ lambda f: signed_face_angle(f['landmarks']) < -MIN_FACE_ANGLE # is_left_profile ] }, ], 'edges': [{ 'start': 'left', 'end': 'right', 'predicates': [ lambda l, r: mouths_are_close(l['landmarks'], r['landmarks']), ] }] } mf_up_close = faces.filter( payload_satisfies(scene_graph(graph, exact=True))).map( map_payload(get_landmarks)).filter( payload_satisfies(scene_graph(graph2, exact=True))) vids = mf_up_close.get_allintervals().keys() # Merge with shots shots_qs = Shot.objects.filter( video_id__in=vids, labeler=Labeler.objects.get(name='shot-hsvhist-face')).all() total = shots_qs.count() print("Total shots:", total) # use emtpy list as payload shots = VideoIntervalCollection.from_django_qs(shots_qs, with_payload=lambda row: [], progress=True, total=total) kissing_shots = mf_up_close.join(shots, lambda kiss, shot: [(kiss.get_start( ), shot.get_end(), kiss.get_payload())], predicate=overlaps(), working_window=1).coalesce() # Getting faces in the shot def wrap_in_list(intvl): intvl.payload = [intvl.payload] return intvl print("Getting faces...") faces_qs2 = Face.objects.filter(frame__video_id__in=vids, probability__gte=MIN_FACE_CONFIDENCE) total = faces_qs2.count() faces2 = VideoIntervalCollection.from_django_qs( faces_qs2.annotate(min_frame=F('frame__number'), max_frame=F('frame__number'), video_id=F('frame__video_id')), with_payload=in_array( merge_dict_parsers([ bbox_payload_parser(VideoIntervalCollection.django_accessor), dict_payload_parser(VideoIntervalCollection.django_accessor, {'frame': 'min_frame'}) ])), progress=True, total=total).coalesce(payload_merge_op=payload_plus).map(wrap_in_list) def clip_to_last_frame_with_two_faces(intvl): faces = intvl.get_payload()[1] two_faces = [(f[0], f[1]) for f in faces if len(f) == 2] two_high_faces = [ (a, b) for a, b in two_faces if min(a['y2'] - a['y1'], b['y2'] - b['y1']) >= MIN_FACE_HEIGHT ] frame = [a['frame'] for a, b in two_high_faces] if len(frame) > 0: intvl.end = frame[-1] return intvl clipped_kissing_shots = kissing_shots.merge( faces2, payload_merge_op=lambda p1, p2: (p1, p2), predicate=overlaps(), working_window=1).coalesce( payload_merge_op=lambda p1, p2: (p1[0], p1[1] + p2[1])).map( clip_to_last_frame_with_two_faces).filter_length(min_length=12) results = get_all_segments(vids) fps_map = dict((i, Video.objects.get(id=i).fps) for i in vids) caption_results = VideoIntervalCollection({ video_id: [ ( word[0] * fps_map[video_id], # start frame word[1] * fps_map[video_id], # end frame word[2]) # payload is the word for word in words ] for video_id, words in results }) kissing_without_words = clipped_kissing_shots.minus(caption_results) kissing_final = kissing_without_words.map(lambda intvl: (int( intvl.start), int(intvl.end), intvl.payload)).coalesce().filter_length( min_length=12) def payload_to_objects(p, video_id): return [face_landmarks_to_dict(face['landmarks']) for face in p[0] ] + [bbox_to_result_object(face, video_id) for face in p[0]] return intrvllists_to_result_with_objects( kissing_final.get_allintervals(), lambda p, vid: payload_to_objects(p, vid), stride=1)
def harry_ron_hermione(): from query.models import FaceCharacterActor from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser from rekall.merge_ops import payload_plus from rekall.payload_predicates import payload_satisfies from rekall.spatial_predicates import scene_graph from rekall.bbox_predicates import height_at_least, left_of, same_value, same_height from esper.rekall import intrvllists_to_result_bbox MIN_FACE_HEIGHT = 0.25 EPSILON = 0.15 NAMES = [ 'ron weasley', 'harry potter', 'hermione granger' ] # Annotate face rows with start and end frames and the video ID faces_with_character_actor_qs = FaceCharacterActor.objects.annotate( min_frame=F('face__frame__number'), max_frame=F('face__frame__number'), video_id=F('face__frame__video_id'), bbox_x1=F('face__bbox_x1'), bbox_y1=F('face__bbox_y1'), bbox_x2=F('face__bbox_x2'), bbox_y2=F('face__bbox_y2'), character_name=F('characteractor__character__name') ).filter(face__frame__video__name__contains="harry potter") faces_with_identity = VideoIntervalCollection.from_django_qs( faces_with_character_actor_qs, with_payload=in_array(merge_dict_parsers([ bbox_payload_parser(VideoIntervalCollection.django_accessor), dict_payload_parser(VideoIntervalCollection.django_accessor, { 'character': 'character_name' }), ])) ).coalesce(payload_merge_op=payload_plus) harry_ron_hermione_scene_graph = { 'nodes': [ { 'name': 'face1', 'predicates': [ height_at_least(MIN_FACE_HEIGHT), lambda f: f['character'] == NAMES[0] ] }, { 'name': 'face2', 'predicates': [ height_at_least(MIN_FACE_HEIGHT), lambda f: f['character'] == NAMES[1] ] }, { 'name': 'face3', 'predicates': [ height_at_least(MIN_FACE_HEIGHT), lambda f: f['character'] == NAMES[2] ] } ], 'edges': [ { 'start': 'face1', 'end': 'face2', 'predicates': [ same_value('y1', epsilon=EPSILON), same_height(epsilon=EPSILON) ] }, { 'start': 'face2', 'end': 'face3', 'predicates': [ same_value('y1', epsilon=EPSILON), same_height(epsilon=EPSILON) ] }, { 'start': 'face1', 'end': 'face3', 'predicates': [ same_value('y1', epsilon=EPSILON), same_height(epsilon=EPSILON) ] } ] } harry_ron_hermione = faces_with_identity.filter(payload_satisfies(scene_graph( harry_ron_hermione_scene_graph, exact=True ))) return intrvllists_to_result_bbox(harry_ron_hermione.get_allintervals(), limit=100, stride=10)
def conversations_for_display(): from query.models import FaceCharacterActor, Shot from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser from rekall.merge_ops import payload_plus from rekall.payload_predicates import payload_satisfies from rekall.spatial_predicates import scene_graph from esper.rekall import intrvllists_to_result_bbox from query.models import Face from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser from rekall.merge_ops import payload_plus, merge_named_payload, payload_second from esper.rekall import intrvllists_to_result_bbox from rekall.payload_predicates import payload_satisfies from rekall.list_predicates import length_at_most from rekall.logical_predicates import and_pred, or_pred, true_pred from rekall.spatial_predicates import scene_graph, make_region from rekall.temporal_predicates import before, after, overlaps, equal from rekall.bbox_predicates import height_at_least from esper.rekall import intrvllists_to_result, intrvllists_to_result_with_objects, add_intrvllists_to_result from esper.prelude import esper_widget from rekall.interval_list import Interval, IntervalList import esper.face_embeddings as face_embeddings video_id = 15 EMBEDDING_EQUALITY_THRESHOLD = 1. ONE_FRAME = 1 faces_qs = Face.objects.annotate(min_frame=F('frame__number'), max_frame=F('frame__number'), video_id=F('frame__video_id')).filter( frame__video_id=video_id, frame__regularly_sampled=True) faces_per_frame = VideoIntervalCollection.from_django_qs( faces_qs, with_payload=in_array( merge_dict_parsers([ bbox_payload_parser(VideoIntervalCollection.django_accessor), dict_payload_parser(VideoIntervalCollection.django_accessor, {'face_id': 'id'}), ]))).coalesce(payload_merge_op=payload_plus) shots_qs = Shot.objects.filter(cinematic=True) shots = VideoIntervalCollection.from_django_qs(shots_qs) shots_with_faces = shots.merge( faces_per_frame, predicate=overlaps(), payload_merge_op=lambda shot_id, faces_in_frame: (shot_id, [faces_in_frame])).coalesce( payload_merge_op=lambda p1, p2: (p1[0], p1[1] + p2[1])) def cluster_center(face_ids): # print("About to compute mean") mean_embedding = face_embeddings.mean(face_ids) # print("About to compute dist", face_ids) dists = face_embeddings.dist(face_ids, [mean_embedding]) # print("Done computing dist") return min(zip(dists, face_ids))[1] def cluster_and_compute_centers(faces_in_frame_list, shot_id): num_people = max( len(faces_in_frame) for faces_in_frame in faces_in_frame_list) face_ids = [ face['face_id'] for faces_in_frame in faces_in_frame_list for face in faces_in_frame ] face_heights = [ face['y2'] - face['y1'] for faces_in_frame in faces_in_frame_list for face in faces_in_frame ] print(num_people) if num_people == 1: clusters = [(fid, 0) for fid in face_ids] else: clusters = face_embeddings.kmeans(face_ids, num_people) # print("Done clustering") centers = [(cluster_center([ face_id for face_id, cluster_id in clusters if cluster_id == i ]), [face_id for face_id, cluster_id in clusters if cluster_id == i], shot_id, max([ face_heights[face_ids.index(face_id)] for face_id, cluster_id in clusters if cluster_id == i ])) for i in range(num_people)] # print("Done computing the center") return centers # print("About to compute clusters") shots_with_centers = shots_with_faces.map(lambda intrvl: ( intrvl.start, intrvl.end, (intrvl.payload[0], cluster_and_compute_centers(intrvl.payload[1], intrvl.payload[0])))) # print("Clusters computed") def same_face(center1, center2): return face_embeddings.dist( [center1], target_ids=[center2])[0] < EMBEDDING_EQUALITY_THRESHOLD def cross_product_faces(intrvl1, intrvl2): payload1 = intrvl1.get_payload() payload2 = intrvl2.get_payload() payload = [] for cluster1 in payload1[1]: for cluster2 in payload2[1]: if not same_face(cluster1[0], cluster2[0]): new_payload = {'A': cluster1, 'B': cluster2} payload.append(new_payload) return [(min(intrvl1.get_start(), intrvl2.get_start()), max(intrvl1.get_end(), intrvl2.get_end()), { 'chrs': payload, 'shots': [payload1[0], payload2[0]] })] two_shots = shots_with_centers.join(shots_with_centers, predicate=after(max_dist=ONE_FRAME, min_dist=ONE_FRAME), merge_op=cross_product_faces) # print("Cross product done") def faces_equal(payload1, payload2): for face_pair1 in payload1['chrs']: for face_pair2 in payload2['chrs']: if (same_face(face_pair1['A'][0], face_pair2['A'][0]) and same_face(face_pair1['B'][0], face_pair2['B'][0])): return True if (same_face(face_pair1['A'][0], face_pair2['B'][0]) and same_face(face_pair1['B'][0], face_pair2['A'][0])): return True return False convs = two_shots.coalesce( predicate=payload_satisfies(faces_equal, arity=2), payload_merge_op=lambda payload1, payload2: { 'chrs': payload1['chrs'] + payload2['chrs'], 'shots': payload1['shots'] + payload2['shots'] }) # print("Coalesce done") adjacent_seq = convs.merge( convs, predicate=and_pred(after(max_dist=ONE_FRAME, min_dist=ONE_FRAME), payload_satisfies(faces_equal, arity=2), arity=2), payload_merge_op=lambda payload1, payload2: { 'chrs': payload1['chrs'] + payload2['chrs'], 'shots': payload1['shots'] + payload2['shots'] }, working_window=1) convs = convs.set_union(adjacent_seq) # convs = convs.coalesce(predicate=times_equal, payload_merge_op=shots_equal) # print("Two-shot adjacencies done") def filter_fn(intvl): payload = intvl.get_payload() if type(payload) is dict and 'shots' in payload: return len(set(payload['shots'])) >= 3 return False convs = convs.filter(filter_fn) convs = convs.coalesce() # print("Final filter done") # for video_id in convs.intervals.keys(): # print(video_id) # intvllist = convs.get_intervallist(video_id) # for intvl in intvllist.get_intervals(): # print(intvl.payload) # print(str(intvl.start) + ':' + str(intvl.end)) return intervallists_to_result_with_objects(convs, lambda a, b: [])
def shot_reverse_shot_intensification(): from query.models import Face from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, named_payload from rekall.merge_ops import payload_plus, merge_named_payload, payload_first from esper.rekall import intrvllists_to_result_bbox from rekall.payload_predicates import payload_satisfies, on_name from rekall.list_predicates import length_at_most from rekall.logical_predicates import and_pred, or_pred from rekall.spatial_predicates import scene_graph, make_region from rekall.temporal_predicates import before, after from rekall.bbox_predicates import height_at_least from esper.rekall import intrvllists_to_result, intrvllists_to_result_with_objects, add_intrvllists_to_result # If True, visualize results in a timeline TIMELINE_OUTPUT = False RIGHT_HALF_MIN_X = 0.45 LEFT_HALF_MAX_X = 0.55 MIN_FACE_HEIGHT = 0.4 MAX_FACES_ON_SCREEN = 2 # faces are sampled every 12 frames SAMPLING_RATE = 12 ONE_SECOND = 24 FOUR_SECONDS = 96 TEN_SECONDS = 240 # Annotate face rows with start and end frames and the video ID faces = Face.objects.annotate(min_frame=F('frame__number'), max_frame=F('frame__number'), video_id=F('frame__video_id'), shot_scale=F('frame__shot_scale')) right_half = make_region(RIGHT_HALF_MIN_X, 0.0, 1.0, 1.0) left_half = make_region(0.0, 0.0, LEFT_HALF_MAX_X, 1.0) graph = { 'nodes': [{ 'name': 'face', 'predicates': [height_at_least(MIN_FACE_HEIGHT)] }], 'edges': [] } vids = VideoIntervalCollection.from_django_qs( faces, with_payload=merge_dict_parsers([ named_payload( 'faces', in_array( bbox_payload_parser( VideoIntervalCollection.django_accessor))), named_payload('shot_scale', in_array(lambda obj: obj.shot_scale)) ])).coalesce( payload_merge_op=merge_named_payload({ 'faces': payload_plus, 'shot_scale': payload_first })) def shot_scales_decreasing(scales): if len(scales) <= 1: return True cur_scale = scales[0] for scale in scales: if cur_scale == 1: cur_scale = scale continue if scale == 1: continue if scale < cur_scale: # Shot scale has gotten father here return False return True # Get sequences where there's a face on the right half of the screen and # there are at most two faces # Payload is the faces in the first frame, and a list of the shot scales # throughout the sequence # Filter out any sequences where the shot scale gets farther away over the sequence faces_on_right = vids.filter( and_pred( payload_satisfies( on_name('faces', length_at_most(MAX_FACES_ON_SCREEN))), payload_satisfies( on_name('faces', scene_graph( graph, region=right_half))))).dilate(SAMPLING_RATE / 2).coalesce( payload_merge_op=merge_named_payload( { 'faces': payload_first, 'shot_scale': payload_plus })).filter(lambda intrvl: shot_scales_decreasing( intrvl.get_payload()['shot_scale'])) # Get sequences where there's a face on the left half of the screen and # there are at most two faces # Payload is the faces in the first frame, and a list of the shot scales # throughout the sequence faces_on_left = vids.filter( and_pred( payload_satisfies( on_name('faces', length_at_most(MAX_FACES_ON_SCREEN))), payload_satisfies( on_name('faces', scene_graph( graph, region=left_half))))).dilate(SAMPLING_RATE / 2).coalesce( payload_merge_op=merge_named_payload( { 'faces': payload_first, 'shot_scale': payload_plus })).filter(lambda intrvl: shot_scales_decreasing( intrvl.get_payload()['shot_scale'])) # Sequences where faces on left up to one second before/after faces on left # Four seconds of buffer time between left-then-right/right-then-left # segments # Filter sequences by decreasing shot sequences # Only keep remaining sequences that last longer than ten seconds shot_reverse_shot_intensification = faces_on_right.merge( faces_on_left, predicate=before(max_dist=ONE_SECOND)).set_union( faces_on_left.merge(faces_on_right, predicate=before(max_dist=ONE_SECOND)) ).dilate(FOUR_SECONDS).coalesce( payload_merge_op=merge_named_payload({ 'faces': payload_first, 'shot_scale': payload_plus })).dilate( -1 * FOUR_SECONDS).filter(lambda intrvl: shot_scales_decreasing( intrvl.get_payload()['shot_scale'])).filter_length( min_length=TEN_SECONDS) def non_uniform(shot_scales): return (len(set(shot_scales)) > 2 if 1 in set(shot_scales) else len(set(shot_scales)) > 1) # Finally, filter out any shot sequences where the shot scales are uniform shot_reverse_shot_intensification = shot_reverse_shot_intensification.filter( lambda intrvl: non_uniform(intrvl.get_payload()['shot_scale'])) # Post-process to display in Esper widget if TIMELINE_OUTPUT: results = intrvllists_to_result( shot_reverse_shot_intensification.get_allintervals()) add_intrvllists_to_result(results, faces_on_left.get_allintervals(), color='black') add_intrvllists_to_result(results, faces_on_right.get_allintervals(), color='green') else: results = intrvllists_to_result_with_objects( shot_reverse_shot_intensification.get_allintervals(), lambda payload, video: []) return results
def hero_shot(): from query.models import Face from rekall.video_interval_collection import VideoIntervalCollection from rekall.parsers import named_payload, in_array, bbox_payload_parser from rekall.parsers import merge_dict_parsers, dict_payload_parser from rekall.merge_ops import payload_plus, payload_first, merge_named_payload from rekall.payload_predicates import payload_satisfies, on_name from rekall.spatial_predicates import scene_graph from rekall.logical_predicates import and_pred from rekall.bbox_predicates import height_at_least, left_of, same_value from esper.rekall import intrvllists_to_result_with_objects, bbox_to_result_object # We're going to look for frames that would be good "hero shot" frames -- # potentially good frames to show in a Netflix preview, for instance. # We're going to look for frames where there's exactly one face of a # certain height, and the frame has certain minimum brightness, # sharpness, and contrast properties. MIN_FACE_HEIGHT = 0.2 MIN_BRIGHTNESS = 50 MIN_SHARPNESS = 50 MIN_CONTRAST = 30 FILM_NAME = "star wars the force awakens" # Annotate face rows with start and end frames, video ID, and frame image # information faces_qs = Face.objects.annotate(min_frame=F('frame__number'), max_frame=F('frame__number'), video_id=F('frame__video_id'), brightness=F('frame__brightness'), contrast=F('frame__contrast'), sharpness=F('frame__sharpness')).filter( frame__video__name=FILM_NAME, brightness__isnull=False, contrast__isnull=False, sharpness__isnull=False) # Load bounding boxes and faces into rekall, and put all faces in one frame faces = VideoIntervalCollection.from_django_qs( faces_qs, with_payload=merge_dict_parsers([ named_payload( 'faces', in_array( bbox_payload_parser( VideoIntervalCollection.django_accessor))), dict_payload_parser( VideoIntervalCollection.django_accessor, { 'brightness': 'brightness', 'contrast': 'contrast', 'sharpness': 'sharpness' }) ])).coalesce( merge_named_payload({ 'faces': payload_plus, 'brightness': payload_first, 'contrast': payload_first, 'sharpness': payload_first })) # Hero shots are shots where there is exactly one face of at least a # certain height, and brightness, contrast, and sharpness are at least # some amount hero_shots = faces.filter( payload_satisfies( and_pred( on_name( 'faces', scene_graph( { 'nodes': [{ 'name': 'face', 'predicates': [height_at_least(MIN_FACE_HEIGHT)] }], 'edges': [] }, exact=True)), lambda payload: (payload['brightness'] > MIN_BRIGHTNESS and payload['contrast'] > MIN_CONTRAST and payload['sharpness'] > MIN_SHARPNESS)))) return intrvllists_to_result_with_objects( hero_shots.get_allintervals(), lambda payload, video_id: [bbox_to_result_object(bbox, video_id) for bbox in payload['faces']], limit=100, stride=10)