Beispiel #1
0
def all_faces_rekall():
    from query.models import Face
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser
    from rekall.merge_ops import payload_plus
    from esper.rekall import intrvllists_to_result_bbox
    from esper.stdlib import qs_to_result

    video_name = "inception"
    # Annotate face rows with start and end frames and the video ID
    faces = Face.objects.filter(frame__video__name=video_name).annotate(
        min_frame=F('frame__number'),
        max_frame=F('frame__number'),
        video_id=F('frame__video_id'))

    # Materialize all the faces and load them into rekall with bounding box payloads
    # Then coalesce them so that all faces in the same frame are in the same interval
    # NOTE that this is slow right now since we're loading all faces!
    vids = VideoIntervalCollection.from_django_qs(
        faces,
        with_payload=in_array(
            bbox_payload_parser(
                VideoIntervalCollection.django_accessor))).coalesce(
                    payload_merge_op=payload_plus)

    # Post-process to display in Esper widget
    return intrvllists_to_result_bbox(vids.get_allintervals(),
                                      limit=100,
                                      stride=100)
Beispiel #2
0
def faces_with_gender():
    from query.models import FaceGender
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
    from rekall.merge_ops import payload_plus
    from esper.rekall import intrvllists_to_result_bbox

    VIDEO_NAME_CONTAINS = "harry potter"

    # Annotate face rows with start and end frames and the video ID
    faces_with_gender = FaceGender.objects.annotate(
        min_frame=F('face__frame__number'),
        max_frame=F('face__frame__number'),
        video_id=F('face__frame__video_id'),
        bbox_x1=F('face__bbox_x1'),
        bbox_y1=F('face__bbox_y1'),
        bbox_x2=F('face__bbox_x2'),
        bbox_y2=F('face__bbox_y2'),
        gender_name=F('gender__name')).filter(
            face__frame__video__name__contains=VIDEO_NAME_CONTAINS)

    faces = VideoIntervalCollection.from_django_qs(
        faces_with_gender,
        with_payload=in_array(
            merge_dict_parsers([
                bbox_payload_parser(VideoIntervalCollection.django_accessor),
                dict_payload_parser(VideoIntervalCollection.django_accessor,
                                    {'gender': 'gender_name'})
            ]))).coalesce(payload_merge_op=payload_plus)

    return intrvllists_to_result_bbox(faces.get_allintervals(),
                                      limit=100,
                                      stride=1000)
Beispiel #3
0
def three_people():
    from query.models import Face
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser
    from rekall.merge_ops import payload_plus
    from esper.rekall import intrvllists_to_result_bbox
    from rekall.payload_predicates import payload_satisfies
    from rekall.spatial_predicates import scene_graph
    from rekall.bbox_predicates import height_at_least, left_of, same_value

    MIN_FACE_HEIGHT = 0.3
    EPSILON = 0.05

    # Annotate face rows with start and end frames and the video ID
    faces = Face.objects.annotate(min_frame=F('frame__number'),
                                  max_frame=F('frame__number'),
                                  video_id=F('frame__video_id'))

    # Materialize all the faces and load them into rekall with bounding box payloads
    # Then coalesce them so that all faces in the same frame are in the same interval
    # NOTE that this is slow right now since we're loading all faces!
    face_lists = VideoIntervalCollection.from_django_qs(
        faces,
        with_payload=in_array(
            bbox_payload_parser(
                VideoIntervalCollection.django_accessor))).coalesce(
                    payload_merge_op=payload_plus)

    three_people_scene_graph = {
        'nodes': [{
            'name': 'face1',
            'predicates': [height_at_least(MIN_FACE_HEIGHT)]
        }, {
            'name': 'face2',
            'predicates': [height_at_least(MIN_FACE_HEIGHT)]
        }, {
            'name': 'face3',
            'predicates': [height_at_least(MIN_FACE_HEIGHT)]
        }],
        'edges': [{
            'start': 'face1',
            'end': 'face2',
            'predicates': [left_of(),
                           same_value('y1', epsilon=EPSILON)]
        }, {
            'start': 'face2',
            'end': 'face3',
            'predicates': [left_of(),
                           same_value('y1', epsilon=EPSILON)]
        }]
    }

    three_people = face_lists.filter(
        payload_satisfies(scene_graph(three_people_scene_graph, exact=True)))

    # Post-process to display in Esper widget
    return intrvllists_to_result_bbox(three_people.get_allintervals(),
                                      limit=100,
                                      stride=100)
Beispiel #4
0
def man_woman_up_close():
    from query.models import FaceGender
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
    from rekall.merge_ops import payload_plus
    from esper.rekall import intrvllists_to_result_bbox
    from rekall.payload_predicates import payload_satisfies
    from rekall.spatial_predicates import scene_graph
    from rekall.bbox_predicates import height_at_least
    
    MIN_FACE_CONFIDENCE = 0.95
    MIN_GENDER_CONFIDENCE = 0.95
    MIN_FACE_HEIGHT = 0.6

    # Annotate face rows with start and end frames and the video ID
    faces_with_gender= FaceGender.objects.filter(face__frame__video__name=video_name).annotate(
        min_frame=F('face__frame__number'),
        max_frame=F('face__frame__number'),
        video_id=F('face__frame__video_id'),
        bbox_x1=F('face__bbox_x1'),
        bbox_y1=F('face__bbox_y1'),
        bbox_x2=F('face__bbox_x2'),
        bbox_y2=F('face__bbox_y2'),
        gender_name=F('gender__name'),
        face_probability=F('face__probability'))

    faces = VideoIntervalCollection.from_django_qs(
        faces_with_gender,
        with_payload=in_array(merge_dict_parsers([
            bbox_payload_parser(VideoIntervalCollection.django_accessor),
            dict_payload_parser(VideoIntervalCollection.django_accessor, { 'gender': 'gender_name' }),
            dict_payload_parser(VideoIntervalCollection.django_accessor, { 'gender_probability': 'probability' }),
            dict_payload_parser(VideoIntervalCollection.django_accessor, { 'face_probability': 'face_probability' })
        ]))
    ).coalesce(payload_merge_op=payload_plus)

    graph = {
        'nodes': [
            { 'name': 'face_male', 'predicates': [
                height_at_least(MIN_FACE_HEIGHT),
                lambda payload: payload['gender'] is 'M',
                lambda payload: payload['face_probability'] > MIN_FACE_CONFIDENCE,
                lambda payload: payload['gender_probability'] > MIN_GENDER_CONFIDENCE
                ] },
            { 'name': 'face_female', 'predicates': [
                height_at_least(MIN_FACE_HEIGHT),
                lambda payload: payload['gender'] is 'F',
                lambda payload: payload['face_probability'] > MIN_FACE_CONFIDENCE,
                lambda payload: payload['gender_probability'] > MIN_GENDER_CONFIDENCE
                ] },
        ],
        'edges': []
    }

    mf_up_close = faces.filter(payload_satisfies(
        scene_graph(graph, exact=True)))

    return intrvllists_to_result_bbox(mf_up_close.get_allintervals(), limit=100, stride=100)
Beispiel #5
0
def panels_rekall():
    from query.models import LabeledCommercial, Face
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser
    from rekall.merge_ops import payload_plus
    from rekall.bbox_predicates import height_at_least, same_value, left_of
    from rekall.spatial_predicates import scene_graph
    from rekall.payload_predicates import payload_satisfies
    from esper.rekall import intrvllists_to_result_bbox

    # Get list of sandbox video IDs
    sandbox_videos = [
        row.video_id
        for row in LabeledCommercial.objects.distinct('video_id')
    ]

    faces_qs = Face.objects.filter(shot__video_id__in=sandbox_videos).annotate(
        video_id=F("shot__video_id"),
        min_frame=F("shot__min_frame"),
        max_frame=F("shot__max_frame")
    )

    # One interval for each face
    faces = VideoIntervalCollection.from_django_qs(
            faces_qs,
            with_payload=in_array(
                bbox_payload_parser(
                    VideoIntervalCollection.django_accessor)))

    # Merge shots
    faces = faces.coalesce(payload_merge_op=payload_plus)

    # Define a scene graph for things that look like panels
    three_faces_scene_graph = {
        'nodes': [
            { 'name': 'face1', 'predicates': [ height_at_least(0.3) ] },
            { 'name': 'face2', 'predicates': [ height_at_least(0.3) ] },
            { 'name': 'face3', 'predicates': [ height_at_least(0.3) ] }
        ],
        'edges': [
            { 'start': 'face1', 'end': 'face2',
                'predicates': [ same_value('y1', epsilon=0.05), left_of() ] }, 
            { 'start': 'face2', 'end': 'face3',
                'predicates': [ same_value('y1', epsilon=0.05), left_of() ] }, 
        ]
    }

    panels = faces.filter(payload_satisfies(
        scene_graph(three_faces_scene_graph, exact=True)
    ))

    return intrvllists_to_result_bbox(panels.get_allintervals())
Beispiel #6
0
def frames_with_character_x():
    from query.models import FaceCharacterActor
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
    from rekall.merge_ops import payload_plus
    from rekall.payload_predicates import payload_satisfies
    from rekall.spatial_predicates import scene_graph
    from esper.rekall import intrvllists_to_result_bbox

    character_name = "harry potter"

    # Annotate face rows with start and end frames and the video ID
    faces_with_character_actor_qs = FaceCharacterActor.objects.annotate(
        min_frame=F('face__frame__number'),
        max_frame=F('face__frame__number'),
        video_id=F('face__frame__video_id'),
        bbox_x1=F('face__bbox_x1'),
        bbox_y1=F('face__bbox_y1'),
        bbox_x2=F('face__bbox_x2'),
        bbox_y2=F('face__bbox_y2'),
        character_name=F('characteractor__character__name'))

    faces_with_identity = VideoIntervalCollection.from_django_qs(
        faces_with_character_actor_qs,
        with_payload=in_array(
            merge_dict_parsers([
                bbox_payload_parser(VideoIntervalCollection.django_accessor),
                dict_payload_parser(VideoIntervalCollection.django_accessor,
                                    {'character': 'character_name'}),
            ]))).coalesce(payload_merge_op=payload_plus)

    faces_with_actor = faces_with_identity.filter(
        payload_satisfies(
            scene_graph({
                'nodes': [{
                    'name':
                    'face1',
                    'predicates': [lambda f: f['character'] == character_name]
                }],
                'edges': []
            })))

    return intrvllists_to_result_bbox(faces_with_actor.get_allintervals(),
                                      limit=100,
                                      stride=1000)
Beispiel #7
0
def get_numface_intrvlcol(relevant_shots, num_face=1):
    faces = Face.objects.filter(shot__in=list(relevant_shots)) \
            .annotate(video_id=F('shot__video_id')) \
            .annotate(min_frame=F('shot__min_frame')) \
            .annotate(max_frame=F('shot__max_frame'))

    # Materialize all the faces and load them into rekall with bounding box payloads
    # Then coalesce them so that all faces in the same frame are in the same interval
    # NOTE that this is slow right now since we're loading all faces!
    numface_intrvlcol = VideoIntervalCollection.from_django_qs(
        faces,
        with_payload=in_array(
            bbox_payload_parser(VideoIntervalCollection.django_accessor))
        ).coalesce(payload_merge_op=payload_plus).filter(payload_satisfies(length_exactly(num_face)))
    
    num_intrvl = 0
    for _, intrvllist in numface_intrvlcol.get_allintervals().items():
        num_intrvl += intrvllist.size()
    print("Get {} relevant {} face intervals".format(num_intrvl, num_face))
    return numface_intrvlcol
Beispiel #8
0
def shot_reverse_shot_complex():
    from query.models import Face, Shot
    from rekall.temporal_predicates import overlaps
    from rekall.merge_ops import payload_second, payload_plus
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.interval_list import Interval, IntervalList
    from rekall.parsers import in_array, bbox_payload_parser
    from rekall.payload_predicates import payload_satisfies
    from rekall.list_predicates import length_at_most
    from rekall.logical_predicates import and_pred
    from rekall.spatial_predicates import scene_graph, make_region
    from rekall.temporal_predicates import before, after
    from rekall.bbox_predicates import height_at_least
    from esper.rekall import intrvllists_to_result_with_objects

    VIDEO_NAME = 'godfather part iii'

    MAX_FACE_MOVEMENT = 0.15
    MIN_FACE_HEIGHT = 0.2
    MAX_FACES_ON_SCREEN = 4
    RIGHT_HALF_MIN_X = 0.33
    LEFT_HALF_MAX_X = 0.66
    SHOTS_LABELER_ID = 64
    # faces are sampled every 12 frames
    SAMPLING_RATE = 12
    # Annotate face rows with start and end frames and the video ID
    faces = Face.objects.annotate(min_frame=F('frame__number'),
                                  max_frame=F('frame__number'),
                                  video_id=F('frame__video_id')).filter(
                                      frame__video__name__contains=VIDEO_NAME)

    shots = VideoIntervalCollection.from_django_qs(Shot.objects.filter(
        video__name__contains=VIDEO_NAME, labeler_id=SHOTS_LABELER_ID),
                                                   with_payload=lambda obj: [])
    # vids are all faces for each frame
    vids = VideoIntervalCollection.from_django_qs(
        faces.filter(probability__gte=0.99),
        with_payload=in_array(
            bbox_payload_parser(
                VideoIntervalCollection.django_accessor))).coalesce(
                    payload_merge_op=payload_plus)

    right_half = make_region(RIGHT_HALF_MIN_X, 0.0, 1.0, 1.0)
    left_half = make_region(0.0, 0.0, LEFT_HALF_MAX_X, 1.0)
    graph = {
        'nodes': [{
            'name': 'face',
            'predicates': [height_at_least(MIN_FACE_HEIGHT)]
        }],
        'edges': []
    }

    faces_on_right = vids.filter(
        and_pred(payload_satisfies(length_at_most(MAX_FACES_ON_SCREEN)),
                 payload_satisfies(scene_graph(graph, region=right_half))))
    faces_on_left = vids.filter(
        and_pred(payload_satisfies(length_at_most(MAX_FACES_ON_SCREEN)),
                 payload_satisfies(scene_graph(graph, region=left_half))))

    def wrap_list(intvl):
        intvl.payload = [intvl.payload]
        return intvl

    def get_height(box):
        return box['y2'] - box['y1']

    def get_center(box):
        return ((box['x1'] + box['x2']) / 2, (box['y1'] + box['y2']) / 2)

    def get_distance(pt1, pt2):
        return np.sqrt((pt1[0] - pt2[0])**2 + (pt1[1] - pt2[1])**2)

    def find_highest_box(boxes):
        if len(boxes) == 0:
            return None
        result = boxes[0]
        best = get_height(result)
        for i in range(1, len(boxes)):
            h = get_height(boxes[i])
            if h > best:
                best = h
                result = boxes[i]
        return result

    def take_highest_in_frame(intvl):
        result = []
        for faces_in_frame in intvl.payload:
            largest = find_highest_box(faces_in_frame)
            if largest is not None:
                result.append(largest)
        intvl.payload = result
        return intvl

    # Check if displacement of box center between frames are within `dist`
    def inter_frame_movement_less_than(dist):
        def check(boxes):
            for b1, b2 in zip(boxes, boxes[1:]):
                if get_distance(get_center(b1), get_center(b2)) > dist:
                    return False
            return True

        return check

    # Payload is a list, each element is a list of faces for a frame
    shots_with_face_on_right = shots.merge(
        faces_on_right, predicate=overlaps(),
        payload_merge_op=payload_second).map(wrap_list).coalesce(
            payload_merge_op=payload_plus).map(take_highest_in_frame).filter(
                payload_satisfies(
                    inter_frame_movement_less_than(MAX_FACE_MOVEMENT)))
    shots_with_face_on_left = shots.merge(
        faces_on_left, predicate=overlaps(),
        payload_merge_op=payload_second).map(wrap_list).coalesce(
            payload_merge_op=payload_plus).map(take_highest_in_frame).filter(
                payload_satisfies(
                    inter_frame_movement_less_than(MAX_FACE_MOVEMENT)))

    # Right-Left-Right sequences
    shot_reverse_shot_1 = shots_with_face_on_right.merge(
        shots_with_face_on_left,
        predicate=before(max_dist=1)).merge(shots_with_face_on_right,
                                            predicate=before(max_dist=1))

    # Left-Right-Left sequences
    shot_reverse_shot_2 = shots_with_face_on_left.merge(
        shots_with_face_on_right,
        predicate=before(max_dist=1)).merge(shots_with_face_on_left,
                                            predicate=before(max_dist=1))

    shot_reverse_shot = shot_reverse_shot_1.set_union(
        shot_reverse_shot_2).coalesce()
    result = intrvllists_to_result_with_objects(
        shot_reverse_shot.get_allintervals(), payload_to_objs=lambda p, v: [])
    return result
Beispiel #9
0
def shot_reverse_shot():
    from query.models import Face
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser
    from rekall.merge_ops import payload_plus
    from esper.rekall import intrvllists_to_result_bbox
    from rekall.payload_predicates import payload_satisfies
    from rekall.list_predicates import length_at_most
    from rekall.logical_predicates import and_pred, or_pred
    from rekall.spatial_predicates import scene_graph, make_region
    from rekall.temporal_predicates import before, after
    from rekall.bbox_predicates import height_at_least
    from esper.rekall import intrvllists_to_result, intrvllists_to_result_with_objects, add_intrvllists_to_result

    # If True, visualize results in a timeline
    TIMELINE_OUTPUT = False

    RIGHT_HALF_MIN_X = 0.45
    LEFT_HALF_MAX_X = 0.55
    MIN_FACE_HEIGHT = 0.4
    MAX_FACES_ON_SCREEN = 2
    # faces are sampled every 12 frames
    SAMPLING_RATE = 12
    ONE_SECOND = 24
    FOUR_SECONDS = 96
    TEN_SECONDS = 240

    # Annotate face rows with start and end frames and the video ID
    faces = Face.objects.annotate(min_frame=F('frame__number'),
                                  max_frame=F('frame__number'),
                                  video_id=F('frame__video_id'))

    right_half = make_region(RIGHT_HALF_MIN_X, 0.0, 1.0, 1.0)
    left_half = make_region(0.0, 0.0, LEFT_HALF_MAX_X, 1.0)

    graph = {
        'nodes': [{
            'name': 'face',
            'predicates': [height_at_least(MIN_FACE_HEIGHT)]
        }],
        'edges': []
    }

    vids = VideoIntervalCollection.from_django_qs(
        faces,
        with_payload=in_array(
            bbox_payload_parser(
                VideoIntervalCollection.django_accessor))).coalesce(
                    payload_merge_op=payload_plus)

    # Get sequences where there's a face on the right half of the screen and
    #   there are at most two faces
    faces_on_right = vids.filter(
        and_pred(payload_satisfies(length_at_most(MAX_FACES_ON_SCREEN)),
                 payload_satisfies(scene_graph(
                     graph, region=right_half)))).dilate(SAMPLING_RATE /
                                                         2).coalesce()

    # Get sequences where there's a face on the left half of the screen and
    #   there are at most two faces
    faces_on_left = vids.filter(
        and_pred(payload_satisfies(length_at_most(MAX_FACES_ON_SCREEN)),
                 payload_satisfies(scene_graph(
                     graph,
                     region=left_half)))).dilate(SAMPLING_RATE / 2).coalesce()

    # Sequences where faces on left up to one second before/after faces on left
    # Four seconds of buffer time between left-then-right/right-then-left
    #   segments
    # Only keep remaining sequences that last longer than ten seconds
    shot_reverse_shot = faces_on_right.merge(
        faces_on_left,
        predicate=or_pred(
            before(max_dist=ONE_SECOND), after(max_dist=ONE_SECOND),
            arity=2)).dilate(FOUR_SECONDS).coalesce().dilate(
                -1 * FOUR_SECONDS).filter_length(min_length=TEN_SECONDS)

    # Post-process to display in Esper widget
    if TIMELINE_OUTPUT:
        results = intrvllists_to_result(shot_reverse_shot.get_allintervals())
        add_intrvllists_to_result(results,
                                  faces_on_left.get_allintervals(),
                                  color='black')
        add_intrvllists_to_result(results,
                                  faces_on_right.get_allintervals(),
                                  color='green')
    else:
        results = intrvllists_to_result_with_objects(
            shot_reverse_shot.get_allintervals(), lambda payload, video: [])
    return results
Beispiel #10
0
def kissing():
    # Takes 7min to run!
    from query.models import Face, Shot
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
    from rekall.merge_ops import payload_plus
    from rekall.payload_predicates import payload_satisfies
    from rekall.spatial_predicates import scene_graph
    from rekall.temporal_predicates import overlaps
    from rekall.face_landmark_predicates import looking_left, looking_right
    from rekall.bbox_predicates import height_at_least, same_height
    import esper.face_landmarks_wrapper as flw
    from esper.captions import get_all_segments
    from esper.rekall import intrvllists_to_result_with_objects, bbox_to_result_object
    from esper.stdlib import face_landmarks_to_dict

    MAX_MOUTH_DIFF = 0.12
    MIN_FACE_CONFIDENCE = 0.8
    MIN_FACE_HEIGHT = 0.4
    MAX_FACE_HEIGHT_DIFF = 0.1
    MIN_FACE_OVERLAP_X = 0.05
    MIN_FACE_OVERLAP_Y = 0.2
    MAX_FACE_OVERLAP_X_FRACTION = 0.7
    MIN_FACE_ANGLE = 0.1

    def map_payload(func):
        def map_fn(intvl):
            intvl.payload = func(intvl.payload)
            return intvl

        return map_fn

    def get_landmarks(faces):
        ids = [face['id'] for face in faces]
        landmarks = flw.get(Face.objects.filter(id__in=ids))
        for face, landmark in zip(faces, landmarks):
            face['landmarks'] = landmark
        return faces

    # Annotate face rows with start and end frames and the video ID
    faces_qs = Face.objects.filter(
        probability__gte=MIN_FACE_CONFIDENCE).annotate(
            min_frame=F('frame__number'),
            max_frame=F('frame__number'),
            height=F('bbox_y2') - F('bbox_y1'),
            video_id=F('frame__video_id')).filter(height__gte=MIN_FACE_HEIGHT)

    faces = VideoIntervalCollection.from_django_qs(
        faces_qs,
        with_payload=in_array(
            merge_dict_parsers([
                bbox_payload_parser(VideoIntervalCollection.django_accessor),
                dict_payload_parser(VideoIntervalCollection.django_accessor,
                                    {'id': 'id'})
            ]))).coalesce(payload_merge_op=payload_plus)

    graph = {
        'nodes': [
            {
                'name': 'face_left',
                'predicates': []
            },
            {
                'name': 'face_right',
                'predicates': []
            },
        ],
        'edges': [
            {
                'start':
                'face_left',
                'end':
                'face_right',
                'predicates': [
                    lambda f1, f2: f1['x2'] < f2['x2'] and f1['x1'] < f2[
                        'x1'],  # Left face on the left
                    lambda f1, f2: f1['x2'] - f2['x1'] >
                    MIN_FACE_OVERLAP_X,  # Faces overlap
                    lambda f1, f2: min(f1['y2'], f2['y2']) - max(
                        f1['y1'], f1['y1']) > MIN_FACE_OVERLAP_Y,
                    lambda f1, f2: f1['y2'] > f2['y1'] and f1['y1'] < f2[
                        'y2'],  # No face is entirely above another
                    same_height(MAX_FACE_HEIGHT_DIFF),
                    lambda f1, f2:
                    (f1['x2'] - f2['x1']) / max(f1['x2'] - f1['x1'], f2[
                        'x2'] - f2['x1']) < MAX_FACE_OVERLAP_X_FRACTION
                ]
            },
        ]
    }

    def mouths_are_close(lm1, lm2):
        select_outer = [2, 3, 4, 8, 9, 10]
        select_inner = [1, 2, 3, 5, 6, 7]
        mouth1 = np.concatenate(
            (lm1.outer_lips()[select_outer], lm1.inner_lips()[select_inner]))
        mouth2 = np.concatenate(
            (lm2.outer_lips()[select_outer], lm2.inner_lips()[select_inner]))
        mean1 = np.mean(mouth1, axis=0)
        mean2 = np.mean(mouth2, axis=0)
        return np.linalg.norm(mean1 - mean2) <= MAX_MOUTH_DIFF

    # Face is profile if both eyes are on the same side of the nose bridge horizontally.
    def is_left_profile(f):
        lm = f['landmarks']
        nose_x = min(lm.nose_bridge()[:, 0])
        left = np.all(lm.left_eye()[:, 0] >= nose_x)
        right = np.all(lm.right_eye()[:, 0] >= nose_x)
        return left and right

    def is_right_profile(f):
        lm = f['landmarks']
        nose_x = max(lm.nose_bridge()[:, 0])
        left = np.all(lm.left_eye()[:, 0] <= nose_x)
        right = np.all(lm.right_eye()[:, 0] <= nose_x)
        return left and right

    # Line is ax+by+c=0
    def project_point_to_line(pt, a, b, c):
        x0, y0 = pt[0], pt[1]
        d = a * a + b * b
        x = (b * (b * x0 - a * y0) - a * c) / d
        y = (a * (-b * x0 + a * y0) - b * c) / d
        return np.array([x, y])

    # Positive if facing right
    def signed_face_angle(lm):
        center_line_indices = [27, 28, 32, 33, 34, 51, 62, 66, 57]
        data = lm.landmarks[center_line_indices]
        fit = np.polyfit(data[:, 0], data[:, 1], 1)
        # y = ax+b
        a, b = fit[0], fit[1]
        A = project_point_to_line(lm.landmarks[center_line_indices[0]], a, -1,
                                  b)
        B = project_point_to_line(lm.landmarks[center_line_indices[-1]], a, -1,
                                  b)
        AB = B - A
        AB = AB / np.linalg.norm(AB)
        C = np.mean(lm.nose_bridge()[2:4], axis=0)
        AC = C - A
        AC = AC / np.linalg.norm(AC)
        return np.cross(AB, AC)

    graph2 = {
        'nodes': [
            {
                'name':
                'left',
                'predicates': [
                    lambda f: signed_face_angle(f['landmarks']) >
                    MIN_FACE_ANGLE
                    #                 is_right_profile
                ]
            },
            {
                'name':
                'right',
                'predicates': [
                    lambda f: signed_face_angle(f['landmarks']) <
                    -MIN_FACE_ANGLE
                    #                 is_left_profile
                ]
            },
        ],
        'edges': [{
            'start':
            'left',
            'end':
            'right',
            'predicates': [
                lambda l, r: mouths_are_close(l['landmarks'], r['landmarks']),
            ]
        }]
    }

    mf_up_close = faces.filter(
        payload_satisfies(scene_graph(graph, exact=True))).map(
            map_payload(get_landmarks)).filter(
                payload_satisfies(scene_graph(graph2, exact=True)))
    vids = mf_up_close.get_allintervals().keys()
    # Merge with shots
    shots_qs = Shot.objects.filter(
        video_id__in=vids,
        labeler=Labeler.objects.get(name='shot-hsvhist-face')).all()
    total = shots_qs.count()
    print("Total shots:", total)
    # use emtpy list as payload
    shots = VideoIntervalCollection.from_django_qs(shots_qs,
                                                   with_payload=lambda row: [],
                                                   progress=True,
                                                   total=total)
    kissing_shots = mf_up_close.join(shots,
                                     lambda kiss, shot: [(kiss.get_start(
                                     ), shot.get_end(), kiss.get_payload())],
                                     predicate=overlaps(),
                                     working_window=1).coalesce()

    # Getting faces in the shot
    def wrap_in_list(intvl):
        intvl.payload = [intvl.payload]
        return intvl

    print("Getting faces...")
    faces_qs2 = Face.objects.filter(frame__video_id__in=vids,
                                    probability__gte=MIN_FACE_CONFIDENCE)
    total = faces_qs2.count()
    faces2 = VideoIntervalCollection.from_django_qs(
        faces_qs2.annotate(min_frame=F('frame__number'),
                           max_frame=F('frame__number'),
                           video_id=F('frame__video_id')),
        with_payload=in_array(
            merge_dict_parsers([
                bbox_payload_parser(VideoIntervalCollection.django_accessor),
                dict_payload_parser(VideoIntervalCollection.django_accessor,
                                    {'frame': 'min_frame'})
            ])),
        progress=True,
        total=total).coalesce(payload_merge_op=payload_plus).map(wrap_in_list)

    def clip_to_last_frame_with_two_faces(intvl):
        faces = intvl.get_payload()[1]
        two_faces = [(f[0], f[1]) for f in faces if len(f) == 2]
        two_high_faces = [
            (a, b) for a, b in two_faces
            if min(a['y2'] - a['y1'], b['y2'] - b['y1']) >= MIN_FACE_HEIGHT
        ]
        frame = [a['frame'] for a, b in two_high_faces]

        if len(frame) > 0:
            intvl.end = frame[-1]
        return intvl

    clipped_kissing_shots = kissing_shots.merge(
        faces2,
        payload_merge_op=lambda p1, p2: (p1, p2),
        predicate=overlaps(),
        working_window=1).coalesce(
            payload_merge_op=lambda p1, p2: (p1[0], p1[1] + p2[1])).map(
                clip_to_last_frame_with_two_faces).filter_length(min_length=12)

    results = get_all_segments(vids)
    fps_map = dict((i, Video.objects.get(id=i).fps) for i in vids)
    caption_results = VideoIntervalCollection({
        video_id: [
            (
                word[0] * fps_map[video_id],  # start frame
                word[1] * fps_map[video_id],  # end frame
                word[2])  # payload is the word
            for word in words
        ]
        for video_id, words in results
    })
    kissing_without_words = clipped_kissing_shots.minus(caption_results)
    kissing_final = kissing_without_words.map(lambda intvl: (int(
        intvl.start), int(intvl.end), intvl.payload)).coalesce().filter_length(
            min_length=12)

    def payload_to_objects(p, video_id):
        return [face_landmarks_to_dict(face['landmarks']) for face in p[0]
                ] + [bbox_to_result_object(face, video_id) for face in p[0]]

    return intrvllists_to_result_with_objects(
        kissing_final.get_allintervals(),
        lambda p, vid: payload_to_objects(p, vid),
        stride=1)
Beispiel #11
0
def harry_ron_hermione():
    from query.models import FaceCharacterActor
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
    from rekall.merge_ops import payload_plus
    from rekall.payload_predicates import payload_satisfies
    from rekall.spatial_predicates import scene_graph
    from rekall.bbox_predicates import height_at_least, left_of, same_value, same_height
    from esper.rekall import intrvllists_to_result_bbox

    MIN_FACE_HEIGHT = 0.25
    EPSILON = 0.15
    NAMES = [ 'ron weasley', 'harry potter', 'hermione granger' ]

    # Annotate face rows with start and end frames and the video ID
    faces_with_character_actor_qs = FaceCharacterActor.objects.annotate(
        min_frame=F('face__frame__number'),
        max_frame=F('face__frame__number'),
        video_id=F('face__frame__video_id'),
        bbox_x1=F('face__bbox_x1'),
        bbox_y1=F('face__bbox_y1'),
        bbox_x2=F('face__bbox_x2'),
        bbox_y2=F('face__bbox_y2'),
        character_name=F('characteractor__character__name')
    ).filter(face__frame__video__name__contains="harry potter")

    faces_with_identity = VideoIntervalCollection.from_django_qs(
        faces_with_character_actor_qs,
        with_payload=in_array(merge_dict_parsers([
            bbox_payload_parser(VideoIntervalCollection.django_accessor),
            dict_payload_parser(VideoIntervalCollection.django_accessor, { 'character': 'character_name' }),
        ]))
    ).coalesce(payload_merge_op=payload_plus)

    harry_ron_hermione_scene_graph = {
        'nodes': [
            { 'name': 'face1', 'predicates': [
                height_at_least(MIN_FACE_HEIGHT),
                lambda f: f['character'] == NAMES[0]
            ] },
            { 'name': 'face2', 'predicates': [
                height_at_least(MIN_FACE_HEIGHT),
                lambda f: f['character'] == NAMES[1]
            ] },
            { 'name': 'face3', 'predicates': [
                height_at_least(MIN_FACE_HEIGHT),
                lambda f: f['character'] == NAMES[2]
            ] }
        ],
        'edges': [
            { 'start': 'face1', 'end': 'face2', 'predicates': [
                same_value('y1', epsilon=EPSILON),
                same_height(epsilon=EPSILON) 
            ] },
            { 'start': 'face2', 'end': 'face3', 'predicates': [
                same_value('y1', epsilon=EPSILON),
                same_height(epsilon=EPSILON) 
            ] },
            { 'start': 'face1', 'end': 'face3', 'predicates': [
                same_value('y1', epsilon=EPSILON),
                same_height(epsilon=EPSILON) 
            ] }
        ]
    }

    harry_ron_hermione = faces_with_identity.filter(payload_satisfies(scene_graph(
        harry_ron_hermione_scene_graph,
        exact=True
    )))

    return intrvllists_to_result_bbox(harry_ron_hermione.get_allintervals(), limit=100, stride=10)
Beispiel #12
0
def conversations_for_display():
    from query.models import FaceCharacterActor, Shot
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
    from rekall.merge_ops import payload_plus
    from rekall.payload_predicates import payload_satisfies
    from rekall.spatial_predicates import scene_graph
    from esper.rekall import intrvllists_to_result_bbox
    from query.models import Face
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser
    from rekall.merge_ops import payload_plus, merge_named_payload, payload_second
    from esper.rekall import intrvllists_to_result_bbox
    from rekall.payload_predicates import payload_satisfies
    from rekall.list_predicates import length_at_most
    from rekall.logical_predicates import and_pred, or_pred, true_pred
    from rekall.spatial_predicates import scene_graph, make_region
    from rekall.temporal_predicates import before, after, overlaps, equal
    from rekall.bbox_predicates import height_at_least
    from esper.rekall import intrvllists_to_result, intrvllists_to_result_with_objects, add_intrvllists_to_result
    from esper.prelude import esper_widget
    from rekall.interval_list import Interval, IntervalList
    import esper.face_embeddings as face_embeddings

    video_id = 15
    EMBEDDING_EQUALITY_THRESHOLD = 1.
    ONE_FRAME = 1

    faces_qs = Face.objects.annotate(min_frame=F('frame__number'),
                                     max_frame=F('frame__number'),
                                     video_id=F('frame__video_id')).filter(
                                         frame__video_id=video_id,
                                         frame__regularly_sampled=True)

    faces_per_frame = VideoIntervalCollection.from_django_qs(
        faces_qs,
        with_payload=in_array(
            merge_dict_parsers([
                bbox_payload_parser(VideoIntervalCollection.django_accessor),
                dict_payload_parser(VideoIntervalCollection.django_accessor,
                                    {'face_id': 'id'}),
            ]))).coalesce(payload_merge_op=payload_plus)

    shots_qs = Shot.objects.filter(cinematic=True)
    shots = VideoIntervalCollection.from_django_qs(shots_qs)

    shots_with_faces = shots.merge(
        faces_per_frame,
        predicate=overlaps(),
        payload_merge_op=lambda shot_id, faces_in_frame:
        (shot_id, [faces_in_frame])).coalesce(
            payload_merge_op=lambda p1, p2: (p1[0], p1[1] + p2[1]))

    def cluster_center(face_ids):
        #         print("About to compute mean")
        mean_embedding = face_embeddings.mean(face_ids)
        #         print("About to compute dist", face_ids)
        dists = face_embeddings.dist(face_ids, [mean_embedding])
        #         print("Done computing dist")
        return min(zip(dists, face_ids))[1]

    def cluster_and_compute_centers(faces_in_frame_list, shot_id):
        num_people = max(
            len(faces_in_frame) for faces_in_frame in faces_in_frame_list)
        face_ids = [
            face['face_id'] for faces_in_frame in faces_in_frame_list
            for face in faces_in_frame
        ]
        face_heights = [
            face['y2'] - face['y1'] for faces_in_frame in faces_in_frame_list
            for face in faces_in_frame
        ]
        print(num_people)
        if num_people == 1:
            clusters = [(fid, 0) for fid in face_ids]
        else:
            clusters = face_embeddings.kmeans(face_ids, num_people)
#         print("Done clustering")
        centers = [(cluster_center([
            face_id for face_id, cluster_id in clusters if cluster_id == i
        ]), [face_id for face_id, cluster_id in clusters
             if cluster_id == i], shot_id,
                    max([
                        face_heights[face_ids.index(face_id)]
                        for face_id, cluster_id in clusters if cluster_id == i
                    ])) for i in range(num_people)]
        #         print("Done computing the center")
        return centers

#     print("About to compute clusters")

    shots_with_centers = shots_with_faces.map(lambda intrvl: (
        intrvl.start, intrvl.end,
        (intrvl.payload[0],
         cluster_and_compute_centers(intrvl.payload[1], intrvl.payload[0]))))

    #     print("Clusters computed")

    def same_face(center1, center2):
        return face_embeddings.dist(
            [center1], target_ids=[center2])[0] < EMBEDDING_EQUALITY_THRESHOLD

    def cross_product_faces(intrvl1, intrvl2):
        payload1 = intrvl1.get_payload()
        payload2 = intrvl2.get_payload()
        payload = []
        for cluster1 in payload1[1]:
            for cluster2 in payload2[1]:
                if not same_face(cluster1[0], cluster2[0]):
                    new_payload = {'A': cluster1, 'B': cluster2}
                    payload.append(new_payload)

        return [(min(intrvl1.get_start(), intrvl2.get_start()),
                 max(intrvl1.get_end(), intrvl2.get_end()), {
                     'chrs': payload,
                     'shots': [payload1[0], payload2[0]]
                 })]

    two_shots = shots_with_centers.join(shots_with_centers,
                                        predicate=after(max_dist=ONE_FRAME,
                                                        min_dist=ONE_FRAME),
                                        merge_op=cross_product_faces)

    #     print("Cross product done")

    def faces_equal(payload1, payload2):
        for face_pair1 in payload1['chrs']:
            for face_pair2 in payload2['chrs']:
                if (same_face(face_pair1['A'][0], face_pair2['A'][0])
                        and same_face(face_pair1['B'][0], face_pair2['B'][0])):
                    return True
                if (same_face(face_pair1['A'][0], face_pair2['B'][0])
                        and same_face(face_pair1['B'][0], face_pair2['A'][0])):
                    return True
        return False

    convs = two_shots.coalesce(
        predicate=payload_satisfies(faces_equal, arity=2),
        payload_merge_op=lambda payload1, payload2: {
            'chrs': payload1['chrs'] + payload2['chrs'],
            'shots': payload1['shots'] + payload2['shots']
        })

    #     print("Coalesce done")

    adjacent_seq = convs.merge(
        convs,
        predicate=and_pred(after(max_dist=ONE_FRAME, min_dist=ONE_FRAME),
                           payload_satisfies(faces_equal, arity=2),
                           arity=2),
        payload_merge_op=lambda payload1, payload2: {
            'chrs': payload1['chrs'] + payload2['chrs'],
            'shots': payload1['shots'] + payload2['shots']
        },
        working_window=1)
    convs = convs.set_union(adjacent_seq)

    # convs = convs.coalesce(predicate=times_equal, payload_merge_op=shots_equal)

    #     print("Two-shot adjacencies done")

    def filter_fn(intvl):
        payload = intvl.get_payload()
        if type(payload) is dict and 'shots' in payload:
            return len(set(payload['shots'])) >= 3
        return False

    convs = convs.filter(filter_fn)
    convs = convs.coalesce()

    #     print("Final filter done")

    #     for video_id in convs.intervals.keys():
    #         print(video_id)
    #         intvllist = convs.get_intervallist(video_id)
    #         for intvl in intvllist.get_intervals():
    #             print(intvl.payload)
    #             print(str(intvl.start) + ':' + str(intvl.end))

    return intervallists_to_result_with_objects(convs, lambda a, b: [])
def shot_reverse_shot_intensification():
    from query.models import Face
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, named_payload
    from rekall.merge_ops import payload_plus, merge_named_payload, payload_first
    from esper.rekall import intrvllists_to_result_bbox
    from rekall.payload_predicates import payload_satisfies, on_name
    from rekall.list_predicates import length_at_most
    from rekall.logical_predicates import and_pred, or_pred
    from rekall.spatial_predicates import scene_graph, make_region
    from rekall.temporal_predicates import before, after
    from rekall.bbox_predicates import height_at_least
    from esper.rekall import intrvllists_to_result, intrvllists_to_result_with_objects, add_intrvllists_to_result

    # If True, visualize results in a timeline
    TIMELINE_OUTPUT = False

    RIGHT_HALF_MIN_X = 0.45
    LEFT_HALF_MAX_X = 0.55
    MIN_FACE_HEIGHT = 0.4
    MAX_FACES_ON_SCREEN = 2
    # faces are sampled every 12 frames
    SAMPLING_RATE = 12
    ONE_SECOND = 24
    FOUR_SECONDS = 96
    TEN_SECONDS = 240

    # Annotate face rows with start and end frames and the video ID
    faces = Face.objects.annotate(min_frame=F('frame__number'),
                                  max_frame=F('frame__number'),
                                  video_id=F('frame__video_id'),
                                  shot_scale=F('frame__shot_scale'))

    right_half = make_region(RIGHT_HALF_MIN_X, 0.0, 1.0, 1.0)
    left_half = make_region(0.0, 0.0, LEFT_HALF_MAX_X, 1.0)

    graph = {
        'nodes': [{
            'name': 'face',
            'predicates': [height_at_least(MIN_FACE_HEIGHT)]
        }],
        'edges': []
    }

    vids = VideoIntervalCollection.from_django_qs(
        faces,
        with_payload=merge_dict_parsers([
            named_payload(
                'faces',
                in_array(
                    bbox_payload_parser(
                        VideoIntervalCollection.django_accessor))),
            named_payload('shot_scale', in_array(lambda obj: obj.shot_scale))
        ])).coalesce(
            payload_merge_op=merge_named_payload({
                'faces': payload_plus,
                'shot_scale': payload_first
            }))

    def shot_scales_decreasing(scales):
        if len(scales) <= 1:
            return True
        cur_scale = scales[0]
        for scale in scales:
            if cur_scale == 1:
                cur_scale = scale
                continue
            if scale == 1:
                continue
            if scale < cur_scale:
                # Shot scale has gotten father here
                return False
        return True

    # Get sequences where there's a face on the right half of the screen and
    #   there are at most two faces
    # Payload is the faces in the first frame, and a list of the shot scales
    #   throughout the sequence
    # Filter out any sequences where the shot scale gets farther away over the sequence
    faces_on_right = vids.filter(
        and_pred(
            payload_satisfies(
                on_name('faces', length_at_most(MAX_FACES_ON_SCREEN))),
            payload_satisfies(
                on_name('faces', scene_graph(
                    graph,
                    region=right_half))))).dilate(SAMPLING_RATE / 2).coalesce(
                        payload_merge_op=merge_named_payload(
                            {
                                'faces': payload_first,
                                'shot_scale': payload_plus
                            })).filter(lambda intrvl: shot_scales_decreasing(
                                intrvl.get_payload()['shot_scale']))

    # Get sequences where there's a face on the left half of the screen and
    #   there are at most two faces
    # Payload is the faces in the first frame, and a list of the shot scales
    #   throughout the sequence
    faces_on_left = vids.filter(
        and_pred(
            payload_satisfies(
                on_name('faces', length_at_most(MAX_FACES_ON_SCREEN))),
            payload_satisfies(
                on_name('faces', scene_graph(
                    graph,
                    region=left_half))))).dilate(SAMPLING_RATE / 2).coalesce(
                        payload_merge_op=merge_named_payload(
                            {
                                'faces': payload_first,
                                'shot_scale': payload_plus
                            })).filter(lambda intrvl: shot_scales_decreasing(
                                intrvl.get_payload()['shot_scale']))

    # Sequences where faces on left up to one second before/after faces on left
    # Four seconds of buffer time between left-then-right/right-then-left
    #   segments
    # Filter sequences by decreasing shot sequences
    # Only keep remaining sequences that last longer than ten seconds
    shot_reverse_shot_intensification = faces_on_right.merge(
        faces_on_left, predicate=before(max_dist=ONE_SECOND)).set_union(
            faces_on_left.merge(faces_on_right,
                                predicate=before(max_dist=ONE_SECOND))
        ).dilate(FOUR_SECONDS).coalesce(
            payload_merge_op=merge_named_payload({
                'faces': payload_first,
                'shot_scale': payload_plus
            })).dilate(
                -1 *
                FOUR_SECONDS).filter(lambda intrvl: shot_scales_decreasing(
                    intrvl.get_payload()['shot_scale'])).filter_length(
                        min_length=TEN_SECONDS)

    def non_uniform(shot_scales):
        return (len(set(shot_scales)) > 2
                if 1 in set(shot_scales) else len(set(shot_scales)) > 1)

    # Finally, filter out any shot sequences where the shot scales are uniform
    shot_reverse_shot_intensification = shot_reverse_shot_intensification.filter(
        lambda intrvl: non_uniform(intrvl.get_payload()['shot_scale']))

    # Post-process to display in Esper widget
    if TIMELINE_OUTPUT:
        results = intrvllists_to_result(
            shot_reverse_shot_intensification.get_allintervals())
        add_intrvllists_to_result(results,
                                  faces_on_left.get_allintervals(),
                                  color='black')
        add_intrvllists_to_result(results,
                                  faces_on_right.get_allintervals(),
                                  color='green')
    else:
        results = intrvllists_to_result_with_objects(
            shot_reverse_shot_intensification.get_allintervals(),
            lambda payload, video: [])
    return results
Beispiel #14
0
def hero_shot():
    from query.models import Face
    from rekall.video_interval_collection import VideoIntervalCollection
    from rekall.parsers import named_payload, in_array, bbox_payload_parser
    from rekall.parsers import merge_dict_parsers, dict_payload_parser
    from rekall.merge_ops import payload_plus, payload_first, merge_named_payload
    from rekall.payload_predicates import payload_satisfies, on_name
    from rekall.spatial_predicates import scene_graph
    from rekall.logical_predicates import and_pred
    from rekall.bbox_predicates import height_at_least, left_of, same_value
    from esper.rekall import intrvllists_to_result_with_objects, bbox_to_result_object

    # We're going to look for frames that would be good "hero shot" frames --
    #   potentially good frames to show in a Netflix preview, for instance.
    # We're going to look for frames where there's exactly one face of a
    #   certain height, and the frame has certain minimum brightness,
    #   sharpness, and contrast properties.
    MIN_FACE_HEIGHT = 0.2
    MIN_BRIGHTNESS = 50
    MIN_SHARPNESS = 50
    MIN_CONTRAST = 30
    FILM_NAME = "star wars the force awakens"

    # Annotate face rows with start and end frames, video ID, and frame image
    #   information
    faces_qs = Face.objects.annotate(min_frame=F('frame__number'),
                                     max_frame=F('frame__number'),
                                     video_id=F('frame__video_id'),
                                     brightness=F('frame__brightness'),
                                     contrast=F('frame__contrast'),
                                     sharpness=F('frame__sharpness')).filter(
                                         frame__video__name=FILM_NAME,
                                         brightness__isnull=False,
                                         contrast__isnull=False,
                                         sharpness__isnull=False)

    # Load bounding boxes and faces into rekall, and put all faces in one frame
    faces = VideoIntervalCollection.from_django_qs(
        faces_qs,
        with_payload=merge_dict_parsers([
            named_payload(
                'faces',
                in_array(
                    bbox_payload_parser(
                        VideoIntervalCollection.django_accessor))),
            dict_payload_parser(
                VideoIntervalCollection.django_accessor, {
                    'brightness': 'brightness',
                    'contrast': 'contrast',
                    'sharpness': 'sharpness'
                })
        ])).coalesce(
            merge_named_payload({
                'faces': payload_plus,
                'brightness': payload_first,
                'contrast': payload_first,
                'sharpness': payload_first
            }))

    # Hero shots are shots where there is exactly one face of at least a
    #   certain height, and brightness, contrast, and sharpness are at least
    #   some amount
    hero_shots = faces.filter(
        payload_satisfies(
            and_pred(
                on_name(
                    'faces',
                    scene_graph(
                        {
                            'nodes': [{
                                'name':
                                'face',
                                'predicates':
                                [height_at_least(MIN_FACE_HEIGHT)]
                            }],
                            'edges': []
                        },
                        exact=True)), lambda payload:
                (payload['brightness'] > MIN_BRIGHTNESS and payload['contrast']
                 > MIN_CONTRAST and payload['sharpness'] > MIN_SHARPNESS))))

    return intrvllists_to_result_with_objects(
        hero_shots.get_allintervals(),
        lambda payload, video_id:
        [bbox_to_result_object(bbox, video_id) for bbox in payload['faces']],
        limit=100,
        stride=10)