def import_vdn_dataset_url(server, url, user): r = requests.get(url) response = r.json() vdn_dataset = create_dataset(response, server) vdn_dataset.save() video = Video() if user: video.uploader = user video.name = vdn_dataset.name video.vdn_dataset = vdn_dataset video.save() if vdn_dataset.download_url: task_name = 'import_vdn_file' import_video_task = TEvent() import_video_task.video = video import_video_task.operation = task_name import_video_task.save() app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) elif vdn_dataset.aws_key and vdn_dataset.aws_bucket: task_name = 'import_vdn_s3' import_video_task = TEvent() import_video_task.video = video import_video_task.operation = task_name import_video_task.save() app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) else: raise NotImplementedError
def launch_task(self, t, created_pk=None): if created_pk: if t.get('video_id', '') == '__pk__': t['video_id'] = created_pk for k, v in t.get('arguments', {}).iteritems(): if v == '__pk__': t['arguments'][k] = created_pk if 'video_id' in t: v = Video.objects.get(pk=t['video_id']) map_filters = get_map_filters(t, v) else: map_filters = [{}] for f in map_filters: args = copy.deepcopy(t.get( 'arguments', {})) # make copy so that spec isnt mutated. if f: if 'filters' not in args: args['filters'] = f else: args['filters'].update(f) dt = TEvent() dt.parent_process = self.process dt.task_group_id = t['task_group_id'] if 'video_id' in t: dt.video_id = t['video_id'] dt.arguments = args dt.queue, op = get_queue_name_and_operation( t['operation'], t.get('arguments', {})) dt.operation = op dt.save() self.task_results[dt.pk] = app.send_task(name=dt.operation, args=[ dt.pk, ], queue=dt.queue)
def handle_downloaded_file(downloaded, video, name, extract=True, user=None, rate=30, rescale=0, ): video.name = name video.save() filename = downloaded.split('/')[-1] if filename.endswith('.dva_export.zip'): create_video_folders(video, create_subdirs=False) os.rename(downloaded, '{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1])) video.uploaded = True video.save() task_name = 'import_video_by_id' import_video_task = TEvent() import_video_task.video = video import_video_task.save() app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) elif filename.endswith('.mp4') or filename.endswith('.flv') or filename.endswith('.zip'): create_video_folders(video, create_subdirs=True) os.rename(downloaded, '{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1])) video.uploaded = True if filename.endswith('.zip'): video.dataset = True video.save() if extract: extract_frames_task = TEvent() extract_frames_task.arguments_json = json.dumps({'rate': rate, 'rescale': rescale}) extract_frames_task.video = video task_name = 'extract_frames' if video.dataset else 'segment_video' extract_frames_task.operation = task_name extract_frames_task.save() app.send_task(name=task_name, args=[extract_frames_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) else: raise ValueError, "Extension {} not allowed".format(filename.split('.')[-1]) return video
def handle_downloaded_file(downloaded, video, name, extract=True, user=None, rate=30, rescale=0): video.name = name video.save() filename = downloaded.split('/')[-1] if filename.endswith('.dva_export.zip'): create_video_folders(video, create_subdirs=False) os.rename(downloaded, '{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1])) video.uploaded = True video.save() operation = 'import_video_by_id' import_video_task = TEvent() import_video_task.video = video import_video_task.save() app.send_task(name=operation, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[operation]) elif filename.endswith('.mp4') or filename.endswith('.flv') or filename.endswith('.zip'): create_video_folders(video, create_subdirs=True) os.rename(downloaded, '{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1])) video.uploaded = True if filename.endswith('.zip'): video.dataset = True video.save() if extract: p = processing.DVAPQLProcess() if video.dataset: query = { 'process_type':DVAPQL.PROCESS, 'tasks':[ { 'arguments':{'rate': rate, 'rescale': rescale,'next_tasks':settings.DEFAULT_PROCESSING_PLAN}, 'video_id':video.pk, 'operation': 'extract_frames', } ] } else: query = { 'process_type':DVAPQL.PROCESS, 'tasks':[ { 'arguments':{'next_tasks':[ {'operation':'decode_video', 'arguments':{ 'rate': rate, 'rescale': rescale, 'next_tasks':settings.DEFAULT_PROCESSING_PLAN } } ]}, 'video_id':video.pk, 'operation': 'segment_video', } ] } p.create_from_json(j=query,user=user) p.launch() else: raise ValueError, "Extension {} not allowed".format(filename.split('.')[-1]) return video
def handle_uploaded_file(f, name, extract=True, user=None, rate=30, rescale=0): video = Video() if user: video.uploader = user video.name = name video.save() primary_key = video.pk filename = f.name filename = filename.lower() if filename.endswith('.dva_export.zip'): create_video_folders(video, create_subdirs=False) with open('{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]), 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) video.uploaded = True video.save() task_name = 'import_video_by_id' import_video_task = TEvent() import_video_task.video = video import_video_task.save() app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) elif filename.endswith('.mp4') or filename.endswith('.flv') or filename.endswith('.zip'): create_video_folders(video, create_subdirs=True) with open('{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]), 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) video.uploaded = True if filename.endswith('.zip'): video.dataset = True video.save() if extract: p = processing.DVAPQLProcess() query = { 'process_type':DVAPQL.PROCESS, 'tasks':[ { 'arguments_json':{'rate': rate, 'rescale': rescale}, 'video_id':video.pk, 'operation': 'extract_frames' if video.dataset else 'segment_video', } ] } p.create_from_json(j=query,user=user) p.launch() else: raise ValueError, "Extension {} not allowed".format(filename.split('.')[-1]) return video
def handle_youtube_video(name, url, extract=True, user=None, perform_scene_detection=True, rate=30, rescale=0): video = Video() if user: video.uploader = user video.name = name video.url = url video.youtube_video = True video.save() task_name = 'extract_frames_by_id' extract_frames_task = TEvent() extract_frames_task.video = video extract_frames_task.operation = task_name extract_frames_task.arguments_json = json.dumps({ 'perform_scene_detection': perform_scene_detection, 'rate': rate, 'rescale': rescale }) extract_frames_task.save() if extract: app.send_task(name=task_name, args=[ extract_frames_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) return video
def delete_video_object(video_pk,deleter,garbage_collection=True): video = Video.objects.get(pk=video_pk) deleted = DeletedVideo() deleted.name = video.name deleted.deleter = deleter deleted.uploader = video.uploader deleted.url = video.url deleted.description = video.description deleted.original_pk = video_pk deleted.save() video.delete() if garbage_collection: delete_task = TEvent() delete_task.arguments_json = json.dumps({'video_pk': video_pk}) delete_task.operation = 'delete_video_by_id' delete_task.save() queue = settings.TASK_NAMES_TO_QUEUE[delete_task.operation] _ = app.send_task(name=delete_task.operation, args=[delete_task.pk], queue=queue)
def import_vdn_dataset_url(server,url,user): r = requests.get(url) response = r.json() vdn_dataset = create_dataset(response, server) vdn_dataset.save() video = Video() if user: video.uploader = user video.name = vdn_dataset.name video.vdn_dataset = vdn_dataset video.save() primary_key = video.pk create_video_folders(video, create_subdirs=False) task_name = 'import_video_by_id' import_video_task = TEvent() import_video_task.video = video import_video_task.save() app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name])
def handle_uploaded_file(f, name, extract=True, user=None, rate=30, rescale=0): video = Video() if user: video.uploader = user video.name = name video.save() primary_key = video.pk filename = f.name filename = filename.lower() if filename.endswith('.dva_export.zip'): create_video_folders(video, create_subdirs=False) with open('{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]), 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) video.uploaded = True video.save() task_name = 'import_video_by_id' import_video_task = TEvent() import_video_task.video = video import_video_task.save() app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) elif filename.endswith('.mp4') or filename.endswith('.flv') or filename.endswith('.zip'): create_video_folders(video, create_subdirs=True) with open('{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]), 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) video.uploaded = True if filename.endswith('.zip'): video.dataset = True video.save() if extract: extract_frames_task = TEvent() extract_frames_task.arguments_json = json.dumps({'rate': rate,'rescale': rescale}) extract_frames_task.video = video task_name = 'extract_frames' if video.dataset else 'segment_video' extract_frames_task.operation = task_name extract_frames_task.save() app.send_task(name=task_name, args=[extract_frames_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) else: raise ValueError, "Extension {} not allowed".format(filename.split('.')[-1]) return video
def import_vdn_detector_url(server, url, user): r = requests.get(url) response = r.json() vdn_detector = create_vdn_detector(response, server) detector = CustomDetector() detector.name = vdn_detector.name detector.vdn_detector = vdn_detector detector.save() if vdn_detector.download_url: task_name = 'import_vdn_detector_file' import_vdn_detector_task = TEvent() import_vdn_detector_task.operation = task_name import_vdn_detector_task.arguments_json = json.dumps({'detector_pk': detector.pk}) import_vdn_detector_task.save() app.send_task(name=task_name, args=[import_vdn_detector_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) elif vdn_detector.aws_key and vdn_detector.aws_bucket: raise NotImplementedError else: raise NotImplementedError
def handle_youtube_video(name, url, extract=True, user=None, rate=30, rescale=0): video = Video() if user: video.uploader = user video.name = name video.url = url video.youtube_video = True video.save() task_name = 'segment_video' extract_frames_task = TEvent() extract_frames_task.video = video extract_frames_task.operation = task_name extract_frames_task.arguments_json = json.dumps({'rate': rate,'rescale': rescale}) extract_frames_task.save() if extract: app.send_task(name=task_name, args=[extract_frames_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) return video
def delete_video_object(video_pk,deleter,garbage_collection=True): video = Video.objects.get(pk=video_pk) deleted = DeletedVideo() deleted.name = video.name deleted.deleter = deleter deleted.uploader = video.uploader deleted.url = video.url deleted.description = video.description deleted.original_pk = video_pk deleted.save() video.delete() if garbage_collection: delete_task = TEvent() delete_task.arguments = {'video_pk': video_pk} delete_task.operation = 'delete_video_by_id' delete_task.save() queue = settings.TASK_NAMES_TO_QUEUE[delete_task.operation] _ = app.send_task(name=delete_task.operation, args=[delete_task.pk], queue=queue)
def create_event(e, v): de = TEvent() de.imported = True de.started = e.get('started', False) de.start_ts = e.get('start_ts', None) de.completed = e.get('completed', False) de.errored = e.get('errored', False) de.error_message = e.get('error_message', "") de.video_id = v.pk de.operation = e.get('operation', "") de.created = e['created'] if 'seconds' in e: de.duration = e.get('seconds', -1) else: de.duration = e.get('duration', -1) de.arguments = e.get('arguments', {}) de.task_id = e.get('task_id', "") return de
def launch_task(self, t): for k, v in t.get('arguments', {}).iteritems(): if (type(v) is str or type(v) is unicode) and v.startswith('__created__'): t['arguments'][k] = self.get_created_object_pk(v) if 'video_id' in t or 'video_selector' in t: if (type(t['video_id']) is str or type(t['video_id']) is unicode) and t['video_id'].startswith( '__created__'): t['video_id'] = self.get_created_object_pk(t['video_id']) v = Video.objects.get(pk=t['video_id']) elif 'video_selector' in t: v = Video.objects.get(**t['video_selector']) t['video_id'] = v.pk else: v = Video.objects.get(pk=t['video_id']) map_filters = get_map_filters(t, v) else: map_filters = [{}] # This is useful in case of perform_stream_capture where batch size is used but number of segments is unknown if map_filters == []: map_filters = [{}] for f in map_filters: args = copy.deepcopy(t.get('arguments', {})) # make copy so that spec isnt mutated. if f: if 'filters' not in args: args['filters'] = f else: args['filters'].update(f) dt = TEvent() dt.parent_process = self.process dt.task_group_id = t['task_group_id'] dt.parent = self.root_task if 'video_id' in t: dt.video_id = t['video_id'] if 'training_set_id' in t: dt.training_set_id = t['training_set_id'] elif 'training_set_selector' in t: dt.training_set_id = TrainingSet.objects.get(**t['training_set_selector']) dt.arguments = args dt.queue, op = get_queue_name_and_operation(t['operation'], t.get('arguments', {})) dt.operation = op dt.save() self.task_results[dt.pk] = app.send_task(name=dt.operation, args=[dt.pk, ], queue=dt.queue)
def import_vdn_dataset_url(server, url, user): r = requests.get(url) response = r.json() vdn_dataset = create_dataset(response, server) vdn_dataset.save() video = Video() if user: video.uploader = user video.name = vdn_dataset.name video.vdn_dataset = vdn_dataset video.save() if vdn_dataset.download_url: task_name = 'import_vdn_file' import_video_task = TEvent() import_video_task.video = video import_video_task.operation = task_name import_video_task.save() app.send_task(name=task_name, args=[ import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) elif vdn_dataset.aws_key and vdn_dataset.aws_bucket: task_name = 'import_vdn_s3' import_video_task = TEvent() import_video_task.video = video import_video_task.operation = task_name import_video_task.save() app.send_task(name=task_name, args=[ import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) else: raise NotImplementedError
def handle_uploaded_file(f, name, extract=True, user=None, perform_scene_detection=True, rate=30, rescale=0): video = Video() if user: video.uploader = user video.name = name video.save() primary_key = video.pk filename = f.name filename = filename.lower() if filename.endswith('.dva_export.zip'): create_video_folders(video, create_subdirs=False) with open( '{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]), 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) video.uploaded = True video.save() task_name = 'import_video_by_id' import_video_task = TEvent() import_video_task.video = video import_video_task.save() app.send_task(name=task_name, args=[ import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) elif filename.endswith('.mp4') or filename.endswith( '.flv') or filename.endswith('.zip'): create_video_folders(video, create_subdirs=True) with open( '{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]), 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) video.uploaded = True if filename.endswith('.zip'): video.dataset = True video.save() if extract: extract_frames_task = TEvent() extract_frames_task.arguments_json = json.dumps({ 'perform_scene_detection': perform_scene_detection, 'rate': rate, 'rescale': rescale }) extract_frames_task.video = video task_name = 'extract_frames_by_id' extract_frames_task.operation = task_name extract_frames_task.save() app.send_task(name=task_name, args=[ extract_frames_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) else: raise ValueError, "Extension {} not allowed".format( filename.split('.')[-1]) return video
def handle_downloaded_file( downloaded, video, name, extract=True, user=None, perform_scene_detection=True, rate=30, rescale=0, ): video.name = name video.save() filename = downloaded.split('/')[-1] if filename.endswith('.dva_export.zip'): create_video_folders(video, create_subdirs=False) os.rename( downloaded, '{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1])) video.uploaded = True video.save() task_name = 'import_video_by_id' import_video_task = TEvent() import_video_task.video = video import_video_task.save() app.send_task(name=task_name, args=[ import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) elif filename.endswith('.mp4') or filename.endswith( '.flv') or filename.endswith('.zip'): create_video_folders(video, create_subdirs=True) os.rename( downloaded, '{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1])) video.uploaded = True if filename.endswith('.zip'): video.dataset = True video.save() if extract: extract_frames_task = TEvent() extract_frames_task.arguments_json = json.dumps({ 'perform_scene_detection': perform_scene_detection, 'rate': rate, 'rescale': rescale }) extract_frames_task.video = video task_name = 'extract_frames_by_id' extract_frames_task.operation = task_name extract_frames_task.save() app.send_task(name=task_name, args=[ extract_frames_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name]) else: raise ValueError, "Extension {} not allowed".format( filename.split('.')[-1]) return video
def create_event(e, v): de = TEvent() de.event_type = e.get('event_type', de.VIDEO) de.started = e.get('started', False) de.completed = e.get('completed', False) de.errored = e.get('errored', False) de.error_message = e.get('error_message', "") de.video_id = v.pk de.operation = e.get('operation', "") de.created = e['created'] de.seconds = e.get('seconds', -1) de.file_name = e.get('file_name', "") de.key = e.get('key', "") de.bucket = e.get('bucket', "") de.requester_pays = e.get('requester_pays', False) de.clustering = e.get('clustering', None) de.arguments = e.get('arguments', {}) de.task_id = e.get('task_id', "") return de
def create_event(e,v): de = TEvent() de.event_type = e.get('event_type',de.VIDEO) de.started = e.get('started',False) de.completed = e.get('completed',False) de.errored = e.get('errored',False) de.error_message = e.get('error_message',"") de.video_id = v.pk de.operation = e.get('operation',"") de.created = e['created'] de.seconds = e.get('seconds',-1) de.file_name = e.get('file_name',"") de.key = e.get('key',"") de.bucket = e.get('bucket',"") de.requester_pays = e.get('requester_pays',False) de.clustering = e.get('clustering',None) de.arguments_json = e.get('arguments_json',"") de.task_id = e.get('task_id',"") return de
def create_event(e, v, dt): de = TEvent() de.imported = dt de.id = e['id'] # id is a uuid de.results = e.get('results', None) de.started = e.get('started', False) de.start_ts = e.get('start_ts', None) # Completed is set to False since we it will be completed only when task performing import uploads all the data. de.completed = False de.errored = e.get('errored', False) de.error_message = e.get('error_message', "") de.video_id = v.pk de.operation = e.get('operation', "") de.created = e['created'] de.min_frame_index = e.get('min_frame_index', None) de.max_frame_index = e.get('max_frame_index', None) de.duration = e.get('duration', -1) de.arguments = e.get('arguments', {}) de.task_id = e.get('task_id', "") return de