# from esper.scannerutil import ScannerWrapper # if True: # db_wrapper = ScannerWrapper.create() db = db_wrapper.db job_config = ScannerJobConfig(io_packet_size=10000, work_packet_size=400, batch=400) hists = run_pipeline(db, videos, batch=job_config.batch, run_opts={ 'io_packet_size': job_config.io_packet_size, 'work_packet_size': job_config.work_packet_size, }) print('hists', len(hists)) hists, videos = unzip([(h, v) for (h, v) in zip(hists, videos) if v.num_frames < 800000]) boundaries = compute_shot_boundaries( db, videos=[v.for_scannertools() for v in videos], db_videos=videos, video_ids=[v.id for v in videos], histograms=hists) # print(len([v for (v, b) in zip(videos, boundaries) if b is None])) Notifier().notify('done')
videos = list(Video.objects.all().order_by('id')) cfg = cluster_config(num_workers=100, worker=worker_config('n1-standard-16', gpu=1), pipelines=[clothing_detection.ClothingDetectionPipeline]) with make_cluster(cfg, sql_pool=2, no_delete=True) as db_wrapper: # if True: # db_wrapper = ScannerWrapper.create() db = db_wrapper.db print('Fetching frames') frames = pcache.get('clothing_frames', lambda: par_for(frames_for_video, videos, workers=8)) videos, frames = unzip([(v, f) for (v, f) in zip(videos, frames) if len(f) > 0]) videos = list(videos) frames = list(frames) videos = videos frames = frames bbox_tables = [ ScannerSQLTable( Face, v, num_elements=len(f), filter= 'MOD(query_frame.number, CAST(FLOOR(query_video.fps * 3) AS INTEGER)) = 0' if v.threeyears_dataset else 'MOD(query_frame.number, CAST(CEIL(query_video.fps * 3) AS INTEGER)) = 0'