output_ops['frame_ids'] = self._sources['frame_ids'].op
return output_ops
detect_faces = FaceDetectionPipeline.make_runner()
videos = list(Video.objects.filter(threeyears_dataset=True).order_by('id'))
if False:
with Timer('benchmark'):
videos = videos[:50]
def run_pipeline(db, videos, frames, **kwargs):
return face_detection.detect_faces(db, videos=[v.for_scannertools() for v in videos], frames=frames, cache=False, **kwargs)
cfg = cluster_config(num_workers=5, worker=worker_config('n1-standard-32'))
configs = [(cfg, [ScannerJobConfig(io_packet_size=1000, work_packet_size=20, batch=1)])]
bench('face', {'videos': videos, 'frames': [[f['number'] for f in Frame.objects.filter(video=v).values('number').order_by('number')] for v in videos]},
run_pipeline, configs, no_delete=True, force=True)
with Timer('run'):
print('Getting frames')
def load_frames():
return [[f['number'] for f in Frame.objects.filter(video=v, shot_boundary=False).values('number').order_by('number')]
for v in tqdm(videos)]
frames = pcache.get('face_frames', load_frames)
cfg = cluster_config(
num_workers=100,
worker=worker_config('n1-standard-64'),
num_load_workers=2,
num_save_workers=2)
# ScannerJobConfig(io_packet_size=20000, work_packet_size=400, batch=400),
# ScannerJobConfig(io_packet_size=10000, work_packet_size=400, batch=400),
# ScannerJobConfig(io_packet_size=10000, work_packet_size=400, batch=40),
# ScannerJobConfig(io_packet_size=10000, work_packet_size=1000, batch=1000)])
# ]
configs = [(cluster_config, [
ScannerJobConfig(io_packet_size=10000,
work_packet_size=400,
batch=400)
])]
bench('hist',
videos,
run_pipeline,
configs,
sample_size=50,
no_delete=True,
force=True)
import sys
import os
@scannerpy.register_python_op()
class BoundariesToJson(scannerpy.Kernel):
def new_stream(self, args):
self._video_id = args['video_id']
print(os.getpid(), 'NEW STREAM {}'.format(self._video_id))
sys.stdout.flush()
videos=[v.for_scannertools() for v in videos],
frames=frames,
faces=[ScannerSQLTable(Face, v) #num_elements=len(f))
for v, f in zip(videos, frames)],
cache=False,
**kwargs)
cfg = cluster_config(
num_workers=5, worker=worker_config('n1-standard-32'), pipelines=[face_embedding.FaceEmbeddingPipeline])
configs = [(cfg, [
ScannerJobConfig(io_packet_size=500, work_packet_size=20, pipelines_per_worker=4),
ScannerJobConfig(io_packet_size=1000, work_packet_size=20, pipelines_per_worker=4),
ScannerJobConfig(io_packet_size=1000, work_packet_size=80, pipelines_per_worker=4),
ScannerJobConfig(io_packet_size=1000, work_packet_size=20, pipelines_per_worker=8),
])]
bench('embedding', {'videos': videos, 'frames': [frames_for_video(v) for v in videos]},
run_pipeline, configs, no_delete=True, force=True)
exit()
videos = list(Video.objects.filter(threeyears_dataset=True).order_by('id'))
def load_frames():
return par_for(frames_for_video, videos, workers=8)
frames = pcache.get('emb_frames', load_frames, force=True)
videos, frames = unzip([(v, f) for (v, f) in zip(videos, frames)
if len(f) > 0])
videos = list(videos)
frames = list(frames)
# Export packed embeddings and IDs into single files
if False:
def get_ids(video):