def all_faces_rekall():
from query.models import Face
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser
from rekall.merge_ops import payload_plus
from esper.rekall import intrvllists_to_result_bbox
from esper.stdlib import qs_to_result
video_name = "inception"
# Annotate face rows with start and end frames and the video ID
faces = Face.objects.filter(frame__video__name=video_name).annotate(
min_frame=F('frame__number'),
max_frame=F('frame__number'),
video_id=F('frame__video_id'))
# Materialize all the faces and load them into rekall with bounding box payloads
# Then coalesce them so that all faces in the same frame are in the same interval
# NOTE that this is slow right now since we're loading all faces!
vids = VideoIntervalCollection.from_django_qs(
faces,
with_payload=in_array(
bbox_payload_parser(
VideoIntervalCollection.django_accessor))).coalesce(
payload_merge_op=payload_plus)
# Post-process to display in Esper widget
return intrvllists_to_result_bbox(vids.get_allintervals(),
limit=100,
stride=100)
def faces_with_gender():
from query.models import FaceGender
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus
from esper.rekall import intrvllists_to_result_bbox
VIDEO_NAME_CONTAINS = "harry potter"
# Annotate face rows with start and end frames and the video ID
faces_with_gender = FaceGender.objects.annotate(
min_frame=F('face__frame__number'),
max_frame=F('face__frame__number'),
video_id=F('face__frame__video_id'),
bbox_x1=F('face__bbox_x1'),
bbox_y1=F('face__bbox_y1'),
bbox_x2=F('face__bbox_x2'),
bbox_y2=F('face__bbox_y2'),
gender_name=F('gender__name')).filter(
face__frame__video__name__contains=VIDEO_NAME_CONTAINS)
faces = VideoIntervalCollection.from_django_qs(
faces_with_gender,
with_payload=in_array(
merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor,
{'gender': 'gender_name'})
]))).coalesce(payload_merge_op=payload_plus)
return intrvllists_to_result_bbox(faces.get_allintervals(),
limit=100,
stride=1000)
def three_people():
from query.models import Face
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser
from rekall.merge_ops import payload_plus
from esper.rekall import intrvllists_to_result_bbox
from rekall.payload_predicates import payload_satisfies
from rekall.spatial_predicates import scene_graph
from rekall.bbox_predicates import height_at_least, left_of, same_value
MIN_FACE_HEIGHT = 0.3
EPSILON = 0.05
# Annotate face rows with start and end frames and the video ID
faces = Face.objects.annotate(min_frame=F('frame__number'),
max_frame=F('frame__number'),
video_id=F('frame__video_id'))
# Materialize all the faces and load them into rekall with bounding box payloads
# Then coalesce them so that all faces in the same frame are in the same interval
# NOTE that this is slow right now since we're loading all faces!
face_lists = VideoIntervalCollection.from_django_qs(
faces,
with_payload=in_array(
bbox_payload_parser(
VideoIntervalCollection.django_accessor))).coalesce(
payload_merge_op=payload_plus)
three_people_scene_graph = {
'nodes': [{
'name': 'face1',
'predicates': [height_at_least(MIN_FACE_HEIGHT)]
}, {
'name': 'face2',
'predicates': [height_at_least(MIN_FACE_HEIGHT)]
}, {
'name': 'face3',
'predicates': [height_at_least(MIN_FACE_HEIGHT)]
}],
'edges': [{
'start': 'face1',
'end': 'face2',
'predicates': [left_of(),
same_value('y1', epsilon=EPSILON)]
}, {
'start': 'face2',
'end': 'face3',
'predicates': [left_of(),
same_value('y1', epsilon=EPSILON)]
}]
}
three_people = face_lists.filter(
payload_satisfies(scene_graph(three_people_scene_graph, exact=True)))
# Post-process to display in Esper widget
return intrvllists_to_result_bbox(three_people.get_allintervals(),
limit=100,
stride=100)
def man_woman_up_close():
from query.models import FaceGender
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus
from esper.rekall import intrvllists_to_result_bbox
from rekall.payload_predicates import payload_satisfies
from rekall.spatial_predicates import scene_graph
from rekall.bbox_predicates import height_at_least
MIN_FACE_CONFIDENCE = 0.95
MIN_GENDER_CONFIDENCE = 0.95
MIN_FACE_HEIGHT = 0.6
# Annotate face rows with start and end frames and the video ID
faces_with_gender= FaceGender.objects.filter(face__frame__video__name=video_name).annotate(
min_frame=F('face__frame__number'),
max_frame=F('face__frame__number'),
video_id=F('face__frame__video_id'),
bbox_x1=F('face__bbox_x1'),
bbox_y1=F('face__bbox_y1'),
bbox_x2=F('face__bbox_x2'),
bbox_y2=F('face__bbox_y2'),
gender_name=F('gender__name'),
face_probability=F('face__probability'))
faces = VideoIntervalCollection.from_django_qs(
faces_with_gender,
with_payload=in_array(merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor, { 'gender': 'gender_name' }),
dict_payload_parser(VideoIntervalCollection.django_accessor, { 'gender_probability': 'probability' }),
dict_payload_parser(VideoIntervalCollection.django_accessor, { 'face_probability': 'face_probability' })
]))
).coalesce(payload_merge_op=payload_plus)
graph = {
'nodes': [
{ 'name': 'face_male', 'predicates': [
height_at_least(MIN_FACE_HEIGHT),
lambda payload: payload['gender'] is 'M',
lambda payload: payload['face_probability'] > MIN_FACE_CONFIDENCE,
lambda payload: payload['gender_probability'] > MIN_GENDER_CONFIDENCE
] },
{ 'name': 'face_female', 'predicates': [
height_at_least(MIN_FACE_HEIGHT),
lambda payload: payload['gender'] is 'F',
lambda payload: payload['face_probability'] > MIN_FACE_CONFIDENCE,
lambda payload: payload['gender_probability'] > MIN_GENDER_CONFIDENCE
] },
],
'edges': []
}
mf_up_close = faces.filter(payload_satisfies(
scene_graph(graph, exact=True)))
return intrvllists_to_result_bbox(mf_up_close.get_allintervals(), limit=100, stride=100)
def panels_rekall():
from query.models import LabeledCommercial, Face
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser
from rekall.merge_ops import payload_plus
from rekall.bbox_predicates import height_at_least, same_value, left_of
from rekall.spatial_predicates import scene_graph
from rekall.payload_predicates import payload_satisfies
from esper.rekall import intrvllists_to_result_bbox
# Get list of sandbox video IDs
sandbox_videos = [
row.video_id
for row in LabeledCommercial.objects.distinct('video_id')
]
faces_qs = Face.objects.filter(shot__video_id__in=sandbox_videos).annotate(
video_id=F("shot__video_id"),
min_frame=F("shot__min_frame"),
max_frame=F("shot__max_frame")
)
# One interval for each face
faces = VideoIntervalCollection.from_django_qs(
faces_qs,
with_payload=in_array(
bbox_payload_parser(
VideoIntervalCollection.django_accessor)))
# Merge shots
faces = faces.coalesce(payload_merge_op=payload_plus)
# Define a scene graph for things that look like panels
three_faces_scene_graph = {
'nodes': [
{ 'name': 'face1', 'predicates': [ height_at_least(0.3) ] },
{ 'name': 'face2', 'predicates': [ height_at_least(0.3) ] },
{ 'name': 'face3', 'predicates': [ height_at_least(0.3) ] }
],
'edges': [
{ 'start': 'face1', 'end': 'face2',
'predicates': [ same_value('y1', epsilon=0.05), left_of() ] },
{ 'start': 'face2', 'end': 'face3',
'predicates': [ same_value('y1', epsilon=0.05), left_of() ] },
]
}
panels = faces.filter(payload_satisfies(
scene_graph(three_faces_scene_graph, exact=True)
))
return intrvllists_to_result_bbox(panels.get_allintervals())
def frames_with_character_x():
from query.models import FaceCharacterActor
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus
from rekall.payload_predicates import payload_satisfies
from rekall.spatial_predicates import scene_graph
from esper.rekall import intrvllists_to_result_bbox
character_name = "harry potter"
# Annotate face rows with start and end frames and the video ID
faces_with_character_actor_qs = FaceCharacterActor.objects.annotate(
min_frame=F('face__frame__number'),
max_frame=F('face__frame__number'),
video_id=F('face__frame__video_id'),
bbox_x1=F('face__bbox_x1'),
bbox_y1=F('face__bbox_y1'),
bbox_x2=F('face__bbox_x2'),
bbox_y2=F('face__bbox_y2'),
character_name=F('characteractor__character__name'))
faces_with_identity = VideoIntervalCollection.from_django_qs(
faces_with_character_actor_qs,
with_payload=in_array(
merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor,
{'character': 'character_name'}),
]))).coalesce(payload_merge_op=payload_plus)
faces_with_actor = faces_with_identity.filter(
payload_satisfies(
scene_graph({
'nodes': [{
'name':
'face1',
'predicates': [lambda f: f['character'] == character_name]
}],
'edges': []
})))
return intrvllists_to_result_bbox(faces_with_actor.get_allintervals(),
limit=100,
stride=1000)
def faces_from_poses_rekall():
from query.models import Pose
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser
from rekall.merge_ops import payload_plus
from esper.rekall import intrvllists_to_result_bbox
from esper.stdlib import qs_to_result
# Annotate pose rows with start and end frames and the video ID
poses = Pose.objects.annotate(min_frame=F('frame__number'),
max_frame=F('frame__number'),
video_id=F('frame__video_id'))
# Parse the pose keypoints and get a bounding box around the face
def get_face_bbox(pose):
pose_keypoints = pose.pose_keypoints()
face_indices = [
Pose.Nose, Pose.Neck, Pose.REye, Pose.LEye, Pose.REar, Pose.LEar
]
x_vals = [
pose_keypoints[index][0] for index in face_indices
if pose_keypoints[index][2] is not 0.0
]
y_vals = [
pose_keypoints[index][1] for index in face_indices
if pose_keypoints[index][2] is not 0.0
]
x1 = min(x_vals)
y1 = min(y_vals)
x2 = max(x_vals)
y2 = max(y_vals)
return {'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2}
# Materialize all the faces and load them into rekall with bounding box payloads
# Then coalesce them so that all faces in the same frame are in the same interval
# NOTE that this is slow right now since we're loading all poses!
vids = VideoIntervalCollection.from_django_qs(poses,
with_payload=in_array(get_face_bbox)) \
.coalesce(payload_merge_op=payload_plus)
# Post-process to display in Esper widget
return intrvllists_to_result_bbox(vids.get_allintervals(),
limit=100,
stride=100)
def harry_ron_hermione():
from query.models import FaceCharacterActor
from rekall.video_interval_collection import VideoIntervalCollection
from rekall.parsers import in_array, bbox_payload_parser, merge_dict_parsers, dict_payload_parser
from rekall.merge_ops import payload_plus
from rekall.payload_predicates import payload_satisfies
from rekall.spatial_predicates import scene_graph
from rekall.bbox_predicates import height_at_least, left_of, same_value, same_height
from esper.rekall import intrvllists_to_result_bbox
MIN_FACE_HEIGHT = 0.25
EPSILON = 0.15
NAMES = [ 'ron weasley', 'harry potter', 'hermione granger' ]
# Annotate face rows with start and end frames and the video ID
faces_with_character_actor_qs = FaceCharacterActor.objects.annotate(
min_frame=F('face__frame__number'),
max_frame=F('face__frame__number'),
video_id=F('face__frame__video_id'),
bbox_x1=F('face__bbox_x1'),
bbox_y1=F('face__bbox_y1'),
bbox_x2=F('face__bbox_x2'),
bbox_y2=F('face__bbox_y2'),
character_name=F('characteractor__character__name')
).filter(face__frame__video__name__contains="harry potter")
faces_with_identity = VideoIntervalCollection.from_django_qs(
faces_with_character_actor_qs,
with_payload=in_array(merge_dict_parsers([
bbox_payload_parser(VideoIntervalCollection.django_accessor),
dict_payload_parser(VideoIntervalCollection.django_accessor, { 'character': 'character_name' }),
]))
).coalesce(payload_merge_op=payload_plus)
harry_ron_hermione_scene_graph = {
'nodes': [
{ 'name': 'face1', 'predicates': [
height_at_least(MIN_FACE_HEIGHT),
lambda f: f['character'] == NAMES[0]
] },
{ 'name': 'face2', 'predicates': [
height_at_least(MIN_FACE_HEIGHT),
lambda f: f['character'] == NAMES[1]
] },
{ 'name': 'face3', 'predicates': [
height_at_least(MIN_FACE_HEIGHT),
lambda f: f['character'] == NAMES[2]
] }
],
'edges': [
{ 'start': 'face1', 'end': 'face2', 'predicates': [
same_value('y1', epsilon=EPSILON),
same_height(epsilon=EPSILON)
] },
{ 'start': 'face2', 'end': 'face3', 'predicates': [
same_value('y1', epsilon=EPSILON),
same_height(epsilon=EPSILON)
] },
{ 'start': 'face1', 'end': 'face3', 'predicates': [
same_value('y1', epsilon=EPSILON),
same_height(epsilon=EPSILON)
] }
]
}
harry_ron_hermione = faces_with_identity.filter(payload_satisfies(scene_graph(
harry_ron_hermione_scene_graph,
exact=True
)))
return intrvllists_to_result_bbox(harry_ron_hermione.get_allintervals(), limit=100, stride=10)