# from esper.scannerutil import ScannerWrapper
# if True:
# db_wrapper = ScannerWrapper.create()
db = db_wrapper.db
job_config = ScannerJobConfig(io_packet_size=10000,
work_packet_size=400,
batch=400)
hists = run_pipeline(db,
videos,
batch=job_config.batch,
run_opts={
'io_packet_size': job_config.io_packet_size,
'work_packet_size':
job_config.work_packet_size,
})
print('hists', len(hists))
hists, videos = unzip([(h, v) for (h, v) in zip(hists, videos)
if v.num_frames < 800000])
boundaries = compute_shot_boundaries(
db,
videos=[v.for_scannertools() for v in videos],
db_videos=videos,
video_ids=[v.id for v in videos],
histograms=hists)
# print(len([v for (v, b) in zip(videos, boundaries) if b is None]))
Notifier().notify('done')
videos = list(Video.objects.all().order_by('id'))
cfg = cluster_config(num_workers=100,
worker=worker_config('n1-standard-16', gpu=1),
pipelines=[clothing_detection.ClothingDetectionPipeline])
with make_cluster(cfg, sql_pool=2, no_delete=True) as db_wrapper:
# if True:
# db_wrapper = ScannerWrapper.create()
db = db_wrapper.db
print('Fetching frames')
frames = pcache.get('clothing_frames',
lambda: par_for(frames_for_video, videos, workers=8))
videos, frames = unzip([(v, f) for (v, f) in zip(videos, frames)
if len(f) > 0])
videos = list(videos)
frames = list(frames)
videos = videos
frames = frames
bbox_tables = [
ScannerSQLTable(
Face,
v,
num_elements=len(f),
filter=
'MOD(query_frame.number, CAST(FLOOR(query_video.fps * 3) AS INTEGER)) = 0'
if v.threeyears_dataset else
'MOD(query_frame.number, CAST(CEIL(query_video.fps * 3) AS INTEGER)) = 0'