def man_woman_up_close():
from query.models import FaceGender
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus
from esper.rekall import intrvllists_to_result_bbox
from rekall.payload_predicates import payload_satisfies
from rekall.spatial_predicates import scene_graph
from rekall.bbox_predicates import height_at_least
MIN_FACE_CONFIDENCE = 0.95
MIN_GENDER_CONFIDENCE = 0.95
MIN_FACE_HEIGHT = 0.6
# Annotate face rows with start and end frames and the video ID
faces_with_gender= FaceGender.objects.filter(face__frame__video__name=video_name).annotate(
min_frame=F('face__frame__number'),
max_frame=F('face__frame__number'),
video_id=F('face__frame__video_id'),
bbox_x1=F('face__bbox_x1'),
bbox_y1=F('face__bbox_y1'),
bbox_x2=F('face__bbox_x2'),
bbox_y2=F('face__bbox_y2'),
gender_name=F('gender__name'),
face_probability=F('face__probability'))
faces = VideoIntervalCollection.from_django_qs(
faces_with_gender,
with_payload=in_array(merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor, { 'gender': 'gender_name' }),
dict_payload_parser(VideoIntervalCollection.django_accessor, { 'gender_probability': 'probability' }),
dict_payload_parser(VideoIntervalCollection.django_accessor, { 'face_probability': 'face_probability' })
]))
).coalesce(payload_merge_op=payload_plus)
graph = {
'nodes': [
{ 'name': 'face_male', 'predicates': [
height_at_least(MIN_FACE_HEIGHT),
lambda payload: payload['gender'] is 'M',
lambda payload: payload['face_probability'] > MIN_FACE_CONFIDENCE,
lambda payload: payload['gender_probability'] > MIN_GENDER_CONFIDENCE
] },
{ 'name': 'face_female', 'predicates': [
height_at_least(MIN_FACE_HEIGHT),
lambda payload: payload['gender'] is 'F',
lambda payload: payload['face_probability'] > MIN_FACE_CONFIDENCE,
lambda payload: payload['gender_probability'] > MIN_GENDER_CONFIDENCE
] },
],
'edges': []
}
mf_up_close = faces.filter(payload_satisfies(
scene_graph(graph, exact=True)))
return intrvllists_to_result_bbox(mf_up_close.get_allintervals(), limit=100, stride=100)
def faces_with_gender():
from query.models import FaceGender
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus
from esper.rekall import intrvllists_to_result_bbox
VIDEO_NAME_CONTAINS = "harry potter"
# Annotate face rows with start and end frames and the video ID
faces_with_gender = FaceGender.objects.annotate(
min_frame=F('face__frame__number'),
max_frame=F('face__frame__number'),
video_id=F('face__frame__video_id'),
bbox_x1=F('face__bbox_x1'),
bbox_y1=F('face__bbox_y1'),
bbox_x2=F('face__bbox_x2'),
bbox_y2=F('face__bbox_y2'),
gender_name=F('gender__name')).filter(
face__frame__video__name__contains=VIDEO_NAME_CONTAINS)
faces = VideoIntervalCollection.from_django_qs(
faces_with_gender,
with_payload=in_array(
merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor,
{'gender': 'gender_name'})
]))).coalesce(payload_merge_op=payload_plus)
return intrvllists_to_result_bbox(faces.get_allintervals(),
limit=100,
stride=1000)
def frames_with_character_x():
from query.models import FaceCharacterActor
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus
from rekall.payload_predicates import payload_satisfies
from rekall.spatial_predicates import scene_graph
from esper.rekall import intrvllists_to_result_bbox
character_name = "harry potter"
# Annotate face rows with start and end frames and the video ID
faces_with_character_actor_qs = FaceCharacterActor.objects.annotate(
min_frame=F('face__frame__number'),
max_frame=F('face__frame__number'),
video_id=F('face__frame__video_id'),
bbox_x1=F('face__bbox_x1'),
bbox_y1=F('face__bbox_y1'),
bbox_x2=F('face__bbox_x2'),
bbox_y2=F('face__bbox_y2'),
character_name=F('characteractor__character__name'))
faces_with_identity = VideoIntervalCollection.from_django_qs(
faces_with_character_actor_qs,
with_payload=in_array(
merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor,
{'character': 'character_name'}),
]))).coalesce(payload_merge_op=payload_plus)
faces_with_actor = faces_with_identity.filter(
payload_satisfies(
scene_graph({
'nodes': [{
'name':
'face1',
'predicates': [lambda f: f['character'] == character_name]
}],
'edges': []
})))
return intrvllists_to_result_bbox(faces_with_actor.get_allintervals(),
limit=100,
stride=1000)
def kissing():
# Takes 7min to run!
from query.models import Face, Shot
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus
from rekall.payload_predicates import payload_satisfies
from rekall.spatial_predicates import scene_graph
from rekall.temporal_predicates import overlaps
from rekall.face_landmark_predicates import looking_left, looking_right
from rekall.bbox_predicates import height_at_least, same_height
import esper.face_landmarks_wrapper as flw
from esper.captions import get_all_segments
from esper.rekall import intrvllists_to_result_with_objects, bbox_to_result_object
from esper.stdlib import face_landmarks_to_dict
MAX_MOUTH_DIFF = 0.12
MIN_FACE_CONFIDENCE = 0.8
MIN_FACE_HEIGHT = 0.4
MAX_FACE_HEIGHT_DIFF = 0.1
MIN_FACE_OVERLAP_X = 0.05
MIN_FACE_OVERLAP_Y = 0.2
MAX_FACE_OVERLAP_X_FRACTION = 0.7
MIN_FACE_ANGLE = 0.1
def map_payload(func):
def map_fn(intvl):
intvl.payload = func(intvl.payload)
return intvl
return map_fn
def get_landmarks(faces):
ids = [face['id'] for face in faces]
landmarks = flw.get(Face.objects.filter(id__in=ids))
for face, landmark in zip(faces, landmarks):
face['landmarks'] = landmark
return faces
# Annotate face rows with start and end frames and the video ID
faces_qs = Face.objects.filter(
probability__gte=MIN_FACE_CONFIDENCE).annotate(
min_frame=F('frame__number'),
max_frame=F('frame__number'),
height=F('bbox_y2') - F('bbox_y1'),
video_id=F('frame__video_id')).filter(height__gte=MIN_FACE_HEIGHT)
faces = VideoIntervalCollection.from_django_qs(
faces_qs,
with_payload=in_array(
merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor,
{'id': 'id'})
]))).coalesce(payload_merge_op=payload_plus)
graph = {
'nodes': [
{
'name': 'face_left',
'predicates': []
},
{
'name': 'face_right',
'predicates': []
},
],
'edges': [
{
'start':
'face_left',
'end':
'face_right',
'predicates': [
lambda f1, f2: f1['x2'] < f2['x2'] and f1['x1'] < f2[
'x1'], # Left face on the left
lambda f1, f2: f1['x2'] - f2['x1'] >
MIN_FACE_OVERLAP_X, # Faces overlap
lambda f1, f2: min(f1['y2'], f2['y2']) - max(
f1['y1'], f1['y1']) > MIN_FACE_OVERLAP_Y,
lambda f1, f2: f1['y2'] > f2['y1'] and f1['y1'] < f2[
'y2'], # No face is entirely above another
same_height(MAX_FACE_HEIGHT_DIFF),
lambda f1, f2:
(f1['x2'] - f2['x1']) / max(f1['x2'] - f1['x1'], f2[
'x2'] - f2['x1']) < MAX_FACE_OVERLAP_X_FRACTION
]
},
]
}
def mouths_are_close(lm1, lm2):
select_outer = [2, 3, 4, 8, 9, 10]
select_inner = [1, 2, 3, 5, 6, 7]
mouth1 = np.concatenate(
(lm1.outer_lips()[select_outer], lm1.inner_lips()[select_inner]))
mouth2 = np.concatenate(
(lm2.outer_lips()[select_outer], lm2.inner_lips()[select_inner]))
mean1 = np.mean(mouth1, axis=0)
mean2 = np.mean(mouth2, axis=0)
return np.linalg.norm(mean1 - mean2) <= MAX_MOUTH_DIFF
# Face is profile if both eyes are on the same side of the nose bridge horizontally.
def is_left_profile(f):
lm = f['landmarks']
nose_x = min(lm.nose_bridge()[:, 0])
left = np.all(lm.left_eye()[:, 0] >= nose_x)
right = np.all(lm.right_eye()[:, 0] >= nose_x)
return left and right
def is_right_profile(f):
lm = f['landmarks']
nose_x = max(lm.nose_bridge()[:, 0])
left = np.all(lm.left_eye()[:, 0] <= nose_x)
right = np.all(lm.right_eye()[:, 0] <= nose_x)
return left and right
# Line is ax+by+c=0
def project_point_to_line(pt, a, b, c):
x0, y0 = pt[0], pt[1]
d = a * a + b * b
x = (b * (b * x0 - a * y0) - a * c) / d
y = (a * (-b * x0 + a * y0) - b * c) / d
return np.array([x, y])
# Positive if facing right
def signed_face_angle(lm):
center_line_indices = [27, 28, 32, 33, 34, 51, 62, 66, 57]
data = lm.landmarks[center_line_indices]
fit = np.polyfit(data[:, 0], data[:, 1], 1)
# y = ax+b
a, b = fit[0], fit[1]
A = project_point_to_line(lm.landmarks[center_line_indices[0]], a, -1,
b)
B = project_point_to_line(lm.landmarks[center_line_indices[-1]], a, -1,
b)
AB = B - A
AB = AB / np.linalg.norm(AB)
C = np.mean(lm.nose_bridge()[2:4], axis=0)
AC = C - A
AC = AC / np.linalg.norm(AC)
return np.cross(AB, AC)
graph2 = {
'nodes': [
{
'name':
'left',
'predicates': [
lambda f: signed_face_angle(f['landmarks']) >
MIN_FACE_ANGLE
# is_right_profile
]
},
{
'name':
'right',
'predicates': [
lambda f: signed_face_angle(f['landmarks']) <
-MIN_FACE_ANGLE
# is_left_profile
]
},
],
'edges': [{
'start':
'left',
'end':
'right',
'predicates': [
lambda l, r: mouths_are_close(l['landmarks'], r['landmarks']),
]
}]
}
mf_up_close = faces.filter(
payload_satisfies(scene_graph(graph, exact=True))).map(
map_payload(get_landmarks)).filter(
payload_satisfies(scene_graph(graph2, exact=True)))
vids = mf_up_close.get_allintervals().keys()
# Merge with shots
shots_qs = Shot.objects.filter(
video_id__in=vids,
labeler=Labeler.objects.get(name='shot-hsvhist-face')).all()
total = shots_qs.count()
print("Total shots:", total)
# use emtpy list as payload
shots = VideoIntervalCollection.from_django_qs(shots_qs,
with_payload=lambda row: [],
progress=True,
total=total)
kissing_shots = mf_up_close.join(shots,
lambda kiss, shot: [(kiss.get_start(
), shot.get_end(), kiss.get_payload())],
predicate=overlaps(),
working_window=1).coalesce()
# Getting faces in the shot
def wrap_in_list(intvl):
intvl.payload = [intvl.payload]
return intvl
print("Getting faces...")
faces_qs2 = Face.objects.filter(frame__video_id__in=vids,
probability__gte=MIN_FACE_CONFIDENCE)
total = faces_qs2.count()
faces2 = VideoIntervalCollection.from_django_qs(
faces_qs2.annotate(min_frame=F('frame__number'),
max_frame=F('frame__number'),
video_id=F('frame__video_id')),
with_payload=in_array(
merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor,
{'frame': 'min_frame'})
])),
progress=True,
total=total).coalesce(payload_merge_op=payload_plus).map(wrap_in_list)
def clip_to_last_frame_with_two_faces(intvl):
faces = intvl.get_payload()[1]
two_faces = [(f[0], f[1]) for f in faces if len(f) == 2]
two_high_faces = [
(a, b) for a, b in two_faces
if min(a['y2'] - a['y1'], b['y2'] - b['y1']) >= MIN_FACE_HEIGHT
]
frame = [a['frame'] for a, b in two_high_faces]
if len(frame) > 0:
intvl.end = frame[-1]
return intvl
clipped_kissing_shots = kissing_shots.merge(
faces2,
payload_merge_op=lambda p1, p2: (p1, p2),
predicate=overlaps(),
working_window=1).coalesce(
payload_merge_op=lambda p1, p2: (p1[0], p1[1] + p2[1])).map(
clip_to_last_frame_with_two_faces).filter_length(min_length=12)
results = get_all_segments(vids)
fps_map = dict((i, Video.objects.get(id=i).fps) for i in vids)
caption_results = VideoIntervalCollection({
video_id: [
(
word[0] * fps_map[video_id], # start frame
word[1] * fps_map[video_id], # end frame
word[2]) # payload is the word
for word in words
]
for video_id, words in results
})
kissing_without_words = clipped_kissing_shots.minus(caption_results)
kissing_final = kissing_without_words.map(lambda intvl: (int(
intvl.start), int(intvl.end), intvl.payload)).coalesce().filter_length(
min_length=12)
def payload_to_objects(p, video_id):
return [face_landmarks_to_dict(face['landmarks']) for face in p[0]
] + [bbox_to_result_object(face, video_id) for face in p[0]]
return intrvllists_to_result_with_objects(
kissing_final.get_allintervals(),
lambda p, vid: payload_to_objects(p, vid),
stride=1)
def harry_ron_hermione():
from query.models import FaceCharacterActor
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus
from rekall.payload_predicates import payload_satisfies
from rekall.spatial_predicates import scene_graph
from rekall.bbox_predicates import height_at_least, left_of, same_value, same_height
from esper.rekall import intrvllists_to_result_bbox
MIN_FACE_HEIGHT = 0.25
EPSILON = 0.15
NAMES = [ 'ron weasley', 'harry potter', 'hermione granger' ]
# Annotate face rows with start and end frames and the video ID
faces_with_character_actor_qs = FaceCharacterActor.objects.annotate(
min_frame=F('face__frame__number'),
max_frame=F('face__frame__number'),
video_id=F('face__frame__video_id'),
bbox_x1=F('face__bbox_x1'),
bbox_y1=F('face__bbox_y1'),
bbox_x2=F('face__bbox_x2'),
bbox_y2=F('face__bbox_y2'),
character_name=F('characteractor__character__name')
).filter(face__frame__video__name__contains="harry potter")
faces_with_identity = VideoIntervalCollection.from_django_qs(
faces_with_character_actor_qs,
with_payload=in_array(merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor, { 'character': 'character_name' }),
]))
).coalesce(payload_merge_op=payload_plus)
harry_ron_hermione_scene_graph = {
'nodes': [
{ 'name': 'face1', 'predicates': [
height_at_least(MIN_FACE_HEIGHT),
lambda f: f['character'] == NAMES[0]
] },
{ 'name': 'face2', 'predicates': [
height_at_least(MIN_FACE_HEIGHT),
lambda f: f['character'] == NAMES[1]
] },
{ 'name': 'face3', 'predicates': [
height_at_least(MIN_FACE_HEIGHT),
lambda f: f['character'] == NAMES[2]
] }
],
'edges': [
{ 'start': 'face1', 'end': 'face2', 'predicates': [
same_value('y1', epsilon=EPSILON),
same_height(epsilon=EPSILON)
] },
{ 'start': 'face2', 'end': 'face3', 'predicates': [
same_value('y1', epsilon=EPSILON),
same_height(epsilon=EPSILON)
] },
{ 'start': 'face1', 'end': 'face3', 'predicates': [
same_value('y1', epsilon=EPSILON),
same_height(epsilon=EPSILON)
] }
]
}
harry_ron_hermione = faces_with_identity.filter(payload_satisfies(scene_graph(
harry_ron_hermione_scene_graph,
exact=True
)))
return intrvllists_to_result_bbox(harry_ron_hermione.get_allintervals(), limit=100, stride=10)
def conversations_for_display():
from query.models import FaceCharacterActor, Shot
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus
from rekall.payload_predicates import payload_satisfies
from rekall.spatial_predicates import scene_graph
from esper.rekall import intrvllists_to_result_bbox
from query.models import Face
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser
from rekall.merge_ops import payload_plus, merge_named_payload, payload_second
from esper.rekall import intrvllists_to_result_bbox
from rekall.payload_predicates import payload_satisfies
from rekall.list_predicates import length_at_most
from rekall.logical_predicates import and_pred, or_pred, true_pred
from rekall.spatial_predicates import scene_graph, make_region
from rekall.temporal_predicates import before, after, overlaps, equal
from rekall.bbox_predicates import height_at_least
from esper.rekall import intrvllists_to_result, intrvllists_to_result_with_objects, add_intrvllists_to_result
from esper.prelude import esper_widget
from rekall.interval_list import Interval, IntervalList
import esper.face_embeddings as face_embeddings
video_id = 15
EMBEDDING_EQUALITY_THRESHOLD = 1.
ONE_FRAME = 1
faces_qs = Face.objects.annotate(min_frame=F('frame__number'),
max_frame=F('frame__number'),
video_id=F('frame__video_id')).filter(
frame__video_id=video_id,
frame__regularly_sampled=True)
faces_per_frame = VideoIntervalCollection.from_django_qs(
faces_qs,
with_payload=in_array(
merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor,
{'face_id': 'id'}),
]))).coalesce(payload_merge_op=payload_plus)
shots_qs = Shot.objects.filter(cinematic=True)
shots = VideoIntervalCollection.from_django_qs(shots_qs)
shots_with_faces = shots.merge(
faces_per_frame,
predicate=overlaps(),
payload_merge_op=lambda shot_id, faces_in_frame:
(shot_id, [faces_in_frame])).coalesce(
payload_merge_op=lambda p1, p2: (p1[0], p1[1] + p2[1]))
def cluster_center(face_ids):
# print("About to compute mean")
mean_embedding = face_embeddings.mean(face_ids)
# print("About to compute dist", face_ids)
dists = face_embeddings.dist(face_ids, [mean_embedding])
# print("Done computing dist")
return min(zip(dists, face_ids))[1]
def cluster_and_compute_centers(faces_in_frame_list, shot_id):
num_people = max(
len(faces_in_frame) for faces_in_frame in faces_in_frame_list)
face_ids = [
face['face_id'] for faces_in_frame in faces_in_frame_list
for face in faces_in_frame
]
face_heights = [
face['y2'] - face['y1'] for faces_in_frame in faces_in_frame_list
for face in faces_in_frame
]
print(num_people)
if num_people == 1:
clusters = [(fid, 0) for fid in face_ids]
else:
clusters = face_embeddings.kmeans(face_ids, num_people)
# print("Done clustering")
centers = [(cluster_center([
face_id for face_id, cluster_id in clusters if cluster_id == i
]), [face_id for face_id, cluster_id in clusters
if cluster_id == i], shot_id,
max([
face_heights[face_ids.index(face_id)]
for face_id, cluster_id in clusters if cluster_id == i
])) for i in range(num_people)]
# print("Done computing the center")
return centers
# print("About to compute clusters")
shots_with_centers = shots_with_faces.map(lambda intrvl: (
intrvl.start, intrvl.end,
(intrvl.payload[0],
cluster_and_compute_centers(intrvl.payload[1], intrvl.payload[0]))))
# print("Clusters computed")
def same_face(center1, center2):
return face_embeddings.dist(
[center1], target_ids=[center2])[0] < EMBEDDING_EQUALITY_THRESHOLD
def cross_product_faces(intrvl1, intrvl2):
payload1 = intrvl1.get_payload()
payload2 = intrvl2.get_payload()
payload = []
for cluster1 in payload1[1]:
for cluster2 in payload2[1]:
if not same_face(cluster1[0], cluster2[0]):
new_payload = {'A': cluster1, 'B': cluster2}
payload.append(new_payload)
return [(min(intrvl1.get_start(), intrvl2.get_start()),
max(intrvl1.get_end(), intrvl2.get_end()), {
'chrs': payload,
'shots': [payload1[0], payload2[0]]
})]
two_shots = shots_with_centers.join(shots_with_centers,
predicate=after(max_dist=ONE_FRAME,
min_dist=ONE_FRAME),
merge_op=cross_product_faces)
# print("Cross product done")
def faces_equal(payload1, payload2):
for face_pair1 in payload1['chrs']:
for face_pair2 in payload2['chrs']:
if (same_face(face_pair1['A'][0], face_pair2['A'][0])
and same_face(face_pair1['B'][0], face_pair2['B'][0])):
return True
if (same_face(face_pair1['A'][0], face_pair2['B'][0])
and same_face(face_pair1['B'][0], face_pair2['A'][0])):
return True
return False
convs = two_shots.coalesce(
predicate=payload_satisfies(faces_equal, arity=2),
payload_merge_op=lambda payload1, payload2: {
'chrs': payload1['chrs'] + payload2['chrs'],
'shots': payload1['shots'] + payload2['shots']
})
# print("Coalesce done")
adjacent_seq = convs.merge(
convs,
predicate=and_pred(after(max_dist=ONE_FRAME, min_dist=ONE_FRAME),
payload_satisfies(faces_equal, arity=2),
arity=2),
payload_merge_op=lambda payload1, payload2: {
'chrs': payload1['chrs'] + payload2['chrs'],
'shots': payload1['shots'] + payload2['shots']
},
working_window=1)
convs = convs.set_union(adjacent_seq)
# convs = convs.coalesce(predicate=times_equal, payload_merge_op=shots_equal)
# print("Two-shot adjacencies done")
def filter_fn(intvl):
payload = intvl.get_payload()
if type(payload) is dict and 'shots' in payload:
return len(set(payload['shots'])) >= 3
return False
convs = convs.filter(filter_fn)
convs = convs.coalesce()
# print("Final filter done")
# for video_id in convs.intervals.keys():
# print(video_id)
# intvllist = convs.get_intervallist(video_id)
# for intvl in intvllist.get_intervals():
# print(intvl.payload)
# print(str(intvl.start) + ':' + str(intvl.end))
return intervallists_to_result_with_objects(convs, lambda a, b: [])
def reaction_shots_apollo_13():
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.merge_ops import payload_plus
from rekall.payload_predicates import payload_satisfies
from rekall.temporal_predicates import overlaps
from rekall.parsers import in_array, merge_dict_parsers, bbox_payload_parser, dict_payload_parser
from esper.caption_metadata import caption_metadata_for_video
from esper.captions import get_all_segments
from esper.rekall import intrvllists_to_result_with_objects
from query.models import FaceCharacterActor, Shot
videos = Video.objects.filter(name__contains="apollo 13").all()
# Load script data
metadata = VideoIntervalCollection({
video.id: caption_metadata_for_video(video.id)
for video in videos
}).filter(lambda meta_interval:
(meta_interval.payload['speaker'] is not None and "man's voice"
not in meta_interval.payload['speaker'] and meta_interval.
payload['speaker'].strip() != "gene krantz"))
all_segments = get_all_segments([video.id for video in videos])
captions_interval_collection = VideoIntervalCollection(
{video: intervals
for video, intervals in all_segments})
captions_with_speaker_id = captions_interval_collection.overlaps(
metadata.filter(payload_satisfies(lambda p: p['aligned'])),
payload_merge_op=lambda word, script_meta:
(word[0], script_meta['speaker']))
# Annotate face rows with start and end frames and the video ID
faces_with_character_actor_qs = FaceCharacterActor.objects.annotate(
min_frame=F('face__frame__number'),
max_frame=F('face__frame__number'),
video_id=F('face__frame__video_id'),
character_name=F('characteractor__character__name')).filter(
video_id__in=[v.id for v in videos])
frames_with_identity = VideoIntervalCollection.from_django_qs(
faces_with_character_actor_qs,
with_payload=in_array(
dict_payload_parser(VideoIntervalCollection.django_accessor,
{'character': 'character_name'}), )).coalesce(
payload_merge_op=payload_plus)
# Annotate shots with all the people in them
shots_qs = Shot.objects.filter(
cinematic=True,
video_id__in=[v.id for v in videos]).annotate(fps=F('video__fps'))
shots = VideoIntervalCollection.from_django_qs(
shots_qs, with_payload=lambda shot: shot.fps)
# Annotate shots with mode shot scale
frames_with_shot_scale_qs = Frame.objects.filter(
regularly_sampled=True,
video_id__in=[v.id for v in videos
]).annotate(min_frame=F('number'),
max_frame=F('number'),
shot_scale_name=F('shot_scale__name')).all()
frames_with_shot_scale = VideoIntervalCollection.from_django_qs(
frames_with_shot_scale_qs, with_payload=lambda f: f.shot_scale_name)
def get_mode(items):
return max(set(items), key=items.count)
shots_with_scale = shots.merge(
frames_with_shot_scale,
predicate=overlaps(),
payload_merge_op=lambda shot_fps, shot_scale: [(shot_fps, shot_scale)]
).coalesce(payload_merge_op=payload_plus).map(
lambda intrvl: (intrvl.start, intrvl.end, {
'fps': intrvl.payload[0][0],
'shot_scale': get_mode([p[1] for p in intrvl.payload])
}))
shots_with_people_in_them = shots_with_scale.overlaps(
frames_with_identity,
payload_merge_op=lambda shot_payload, identities:
(shot_payload, identities),
working_window=1).coalesce(payload_merge_op=lambda p1, p2: (p1[0], p1[
1] + p2[1])).map(lambda intrvl: (intrvl.start / intrvl.payload[0][
'fps'], intrvl.end / intrvl.payload[0]['fps'], {
'fps':
intrvl.payload[0]['fps'],
'shot_scale':
intrvl.payload[0]['shot_scale'],
'characters':
set([
name.strip().split(' ')[0].strip() for d in intrvl.
payload[1] for name in d['character'].split('/')
if len(name.strip()) > 0
])
}))
reaction_shots = captions_with_speaker_id.overlaps(
shots_with_people_in_them.filter(
payload_satisfies(
lambda p: p['shot_scale'] in
['medium_close_up', 'close_up', 'extreme_close_up'])),
predicate=lambda captions, shots: captions.payload[1].strip().split(
' ')[0] not in shots.payload['characters'],
payload_merge_op=lambda word_and_speaker, fps_and_characters:
(fps_and_characters['fps'], word_and_speaker)).map(lambda intrvl: (
int(intrvl.start * intrvl.payload[0]),
int(intrvl.end * intrvl.payload[0]), [intrvl.payload[1]])).dilate(
12).coalesce(
payload_merge_op=payload_plus).dilate(-12).filter_length(
min_length=12)
return intrvllists_to_result_with_objects(reaction_shots, lambda a, b: [])
def hero_shot():
from query.models import Face
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import named_payload, in_array, bbox_payload_parser
from rekall.parsers import merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus, payload_first, merge_named_payload
from rekall.payload_predicates import payload_satisfies, on_name
from rekall.spatial_predicates import scene_graph
from rekall.logical_predicates import and_pred
from rekall.bbox_predicates import height_at_least, left_of, same_value
from esper.rekall import intrvllists_to_result_with_objects, bbox_to_result_object
# We're going to look for frames that would be good "hero shot" frames --
# potentially good frames to show in a Netflix preview, for instance.
# We're going to look for frames where there's exactly one face of a
# certain height, and the frame has certain minimum brightness,
# sharpness, and contrast properties.
MIN_FACE_HEIGHT = 0.2
MIN_BRIGHTNESS = 50
MIN_SHARPNESS = 50
MIN_CONTRAST = 30
FILM_NAME = "star wars the force awakens"
# Annotate face rows with start and end frames, video ID, and frame image
# information
faces_qs = Face.objects.annotate(min_frame=F('frame__number'),
max_frame=F('frame__number'),
video_id=F('frame__video_id'),
brightness=F('frame__brightness'),
contrast=F('frame__contrast'),
sharpness=F('frame__sharpness')).filter(
frame__video__name=FILM_NAME,
brightness__isnull=False,
contrast__isnull=False,
sharpness__isnull=False)
# Load bounding boxes and faces into rekall, and put all faces in one frame
faces = VideoIntervalCollection.from_django_qs(
faces_qs,
with_payload=merge_dict_parsers([
named_payload(
'faces',
in_array(
bbox_payload_parser(
VideoIntervalCollection.django_accessor))),
dict_payload_parser(
VideoIntervalCollection.django_accessor, {
'brightness': 'brightness',
'contrast': 'contrast',
'sharpness': 'sharpness'
})
])).coalesce(
merge_named_payload({
'faces': payload_plus,
'brightness': payload_first,
'contrast': payload_first,
'sharpness': payload_first
}))
# Hero shots are shots where there is exactly one face of at least a
# certain height, and brightness, contrast, and sharpness are at least
# some amount
hero_shots = faces.filter(
payload_satisfies(
and_pred(
on_name(
'faces',
scene_graph(
{
'nodes': [{
'name':
'face',
'predicates':
[height_at_least(MIN_FACE_HEIGHT)]
}],
'edges': []
},
exact=True)), lambda payload:
(payload['brightness'] > MIN_BRIGHTNESS and payload['contrast']
> MIN_CONTRAST and payload['sharpness'] > MIN_SHARPNESS))))
return intrvllists_to_result_with_objects(
hero_shots.get_allintervals(),
lambda payload, video_id:
[bbox_to_result_object(bbox, video_id) for bbox in payload['faces']],
limit=100,
stride=10)