def import_vdn_dataset_url(server, url, user, cached_response): response = None try: r = requests.get(url) response = r.json() except: pass if not response: response = cached_response video = Video() video.description = "import from {} : {} ".format(server.url, response['description']) if user: video.uploader = user video.name = response['name'] video.save() if response['download_url']: p = processing.DVAPQLProcess() query = { 'process_type': DVAPQL.PROCESS, 'tasks': [{ 'arguments': { 'source': 'VDN_URL', 'url': response['download_url'] }, 'video_id': video.pk, 'operation': 'perform_import', }] } p.create_from_json(j=query, user=user) p.launch() elif response['aws_key'] and response['aws_bucket']: p = processing.DVAPQLProcess() query = { 'process_type': DVAPQL.PROCESS, 'tasks': [{ 'arguments': { 'source': 'VDN_S3', 'key': response['aws_key'], 'bucket': response['aws_bucket'] }, 'video_id': video.pk, 'operation': 'perform_import', }] } p.create_from_json(j=query, user=user) p.launch() else: raise NotImplementedError
def import_vdn_detector_url(server, url, user, cached_response): response = None try: r = requests.get(url) response = r.json() except: pass if not response: response = cached_response detector = DeepModel() detector.model_type = DeepModel.DETECTOR detector.name = response['name'] detector.detector_type = response.get('detector_type', DeepModel.YOLO) detector.save() if response.get('download_url', False): p = processing.DVAPQLProcess() query = { 'process_type': DVAPQL.PROCESS, 'tasks': [{ 'arguments': { 'detector_pk': detector.pk, 'download_url': response['download_url'] }, 'operation': 'perform_detector_import', }] } p.create_from_json(j=query, user=user) p.launch() elif response.get('aws_key', False) and response.get('aws_bucket', False): raise NotImplementedError else: raise NotImplementedError
def import_vdn_detector_url(server, url, user, cached_response): response = None try: r = requests.get(url) response = r.json() except: pass if not response: response = cached_response p = processing.DVAPQLProcess() query = { 'process_type': DVAPQL.PROCESS, 'create': [{ 'MODEL': 'TrainedModel', 'spec': { 'name': response['name'], 'detector_type': TrainedModel.DETECTOR }, 'tasks': [ { 'operation': 'perform_detector_import', 'arguments': { 'path': response['path'], 'detector_pk': '__pk__' }, }, ] }] } p.create_from_json(j=query, user=user) p.launch()
def qt_lopq(): import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from dvaapp.models import Retriever, DeepModel,TEvent,DVAPQL from dvaapp import processing spec = { 'process_type':DVAPQL.PROCESS, 'create':[{ 'MODEL':'Retriever', 'spec':{ 'algorithm':Retriever.LOPQ, 'arguments':{'components': 32, 'm': 8, 'v': 8, 'sub': 128}, 'source_filters':{'indexer_shasum': DeepModel.objects.get(name="inception",model_type=DeepModel.INDEXER).shasum} }, 'tasks':[ { 'operation':'perform_retriever_creation', 'arguments':{'retriever_pk': '__pk__'} } ] },] } p = processing.DVAPQLProcess() p.create_from_json(j=spec,user=None) p.launch() p.wait()
def import_vdn_dataset_url(server, url, user, cached_response): response = None try: r = requests.get(url) response = r.json() except: pass if not response: response = cached_response p = processing.DVAPQLProcess() query = { 'process_type': DVAPQL.PROCESS, 'create': [ { 'spec': { 'name': response['name'], 'uploader_id': user.pk if user else None, 'url': response['path'], 'created': '__timezone.now__', 'description':"import from {} : {} ".format(server.url,response['description']) }, 'MODEL': 'Video', 'tasks': [ { 'arguments': {'path':response['path']}, 'video_id': '__pk__', 'operation': 'perform_import', } ] }, ], } p.create_from_json(j=query, user=user) p.launch()
def delete_video_object(video_pk, deleter): p = processing.DVAPQLProcess() query = { 'process_type': dvaapp.models.DVAPQL.PROCESS, 'delete': [{ 'MODEL': 'Video', 'selector': { 'pk': video_pk }, }] } p.create_from_json(j=query, user=deleter) p.launch()
def delete_video_object(video_pk,deleter): p = processing.DVAPQLProcess() query = { 'process_type': DVAPQL.PROCESS, 'tasks': [ { 'arguments': {'video_pk': video_pk,'deleter_pk':deleter.pk}, 'operation': 'perform_deletion', } ] } p.create_from_json(j=query, user=deleter) p.launch()
def handle_uploaded_file(f, name, extract=True, user=None, rate=None, rescale=None): if rate is None: rate = defaults.DEFAULT_RATE if rescale is None: rescale = defaults.DEFAULT_RESCALE video = Video() if user: video.uploader = user video.name = name video.save() filename = f.name filename = filename.lower() if filename.endswith('.dva_export.zip'): video.create_directory(create_subdirs=False) with open( '{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]), 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) if settings.DISABLE_NFS or settings.HEROKU_DEPLOY: fpath = '/{}/{}.{}'.format(video.pk, video.pk, filename.split('.')[-1]) fs.upload_file_to_remote(fpath) os.remove('{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1])) video.uploaded = True video.save() p = processing.DVAPQLProcess() query = { 'process_type': DVAPQL.PROCESS, 'tasks': [{ 'arguments': { 'source': 'LOCAL' }, 'video_id': video.pk, 'operation': 'perform_import', }] } p.create_from_json(j=query, user=user) p.launch() elif filename.endswith('.mp4') or filename.endswith( '.flv') or filename.endswith('.zip'): video.create_directory(create_subdirs=True) with open( '{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]), 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) if settings.DISABLE_NFS or settings.HEROKU_DEPLOY: fpath = '/{}/video/{}.{}'.format(video.pk, video.pk, filename.split('.')[-1]) fs.upload_file_to_remote(fpath) os.remove('{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1])) video.uploaded = True if filename.endswith('.zip'): video.dataset = True video.save() if extract: p = processing.DVAPQLProcess() if video.dataset: query = { 'process_type': DVAPQL.PROCESS, 'tasks': [{ 'arguments': { 'rescale': rescale, 'next_tasks': defaults.DEFAULT_PROCESSING_PLAN_DATASET }, 'video_id': video.pk, 'operation': 'perform_dataset_extraction', }] } else: query = { 'process_type': DVAPQL.PROCESS, 'tasks': [{ 'arguments': { 'next_tasks': [{ 'operation': 'perform_video_decode', 'arguments': { 'segments_batch_size': defaults.DEFAULT_SEGMENTS_BATCH_SIZE, 'rate': rate, 'rescale': rescale, 'next_tasks': defaults.DEFAULT_PROCESSING_PLAN_VIDEO } }] }, 'video_id': video.pk, 'operation': 'perform_video_segmentation', }] } p.create_from_json(j=query, user=user) p.launch() else: raise ValueError, "Extension {} not allowed".format( filename.split('.')[-1]) return video
def handle_uploaded_file(f, name, user=None, rate=None): if rate is None: rate = defaults.DEFAULT_RATE filename = f.name filename = filename.lower() vuid = str(uuid.uuid1()).replace('-', '_') extension = filename.split('.')[-1] if filename.endswith('.dva_export.zip'): local_fname = '{}/ingest/{}.dva_export.zip'.format( settings.MEDIA_ROOT, vuid) fpath = '/ingest/{}.dva_export.zip'.format(vuid) with open(local_fname, 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) if settings.DISABLE_NFS: fs.upload_file_to_remote(fpath) os.remove(local_fname) p = processing.DVAPQLProcess() query = { 'process_type': DVAPQL.PROCESS, 'create': [ { 'spec': { 'name': name, 'uploader_id': user.pk if user else None, 'created': '__timezone.now__' }, 'MODEL': 'Video', 'tasks': [{ 'arguments': { 'path': fpath }, 'video_id': '__pk__', 'operation': 'perform_import', }] }, ], } p.create_from_json(j=query, user=user) p.launch() elif extension in ['zip', 'gz', 'json', 'mp4']: local_fname = '{}/ingest/{}.{}'.format(settings.MEDIA_ROOT, vuid, filename.split('.')[-1]) fpath = '/ingest/{}.{}'.format(vuid, filename.split('.')[-1]) with open(local_fname, 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) if settings.DISABLE_NFS: fs.upload_file_to_remote(fpath) os.remove(local_fname) p = processing.DVAPQLProcess() if extension == 'zip': query = { 'process_type': DVAPQL.PROCESS, 'create': [ { 'spec': { 'name': name, 'dataset': True, 'uploader_id': user.pk if user else None, 'created': '__timezone.now__' }, 'MODEL': 'Video', 'tasks': [{ 'arguments': { 'path': fpath, 'map': [{ 'arguments': { 'map': json.load( file( "../configs/custom_defaults/dataset_processing.json" )) }, 'operation': 'perform_dataset_extraction', }] }, 'video_id': '__pk__', 'operation': 'perform_import' }] }, ], } elif extension == 'json' or extension == 'gz': query = { 'process_type': DVAPQL.PROCESS, 'create': [ { 'spec': { 'name': name, 'dataset': True, 'uploader_id': user.pk if user else None, 'created': '__timezone.now__' }, 'MODEL': 'Video', 'tasks': [{ 'arguments': { 'path': fpath, 'map': [{ 'operation': 'perform_frame_download', 'arguments': { 'frames_batch_size': defaults.DEFAULT_FRAMES_BATCH_SIZE, 'map': json.load( file( "../configs/custom_defaults/framelist_processing.json" )) }, }] }, 'video_id': '__pk__', 'operation': 'perform_import' }] }, ], } else: query = { 'process_type': DVAPQL.PROCESS, 'create': [ { 'spec': { 'name': name, 'uploader_id': user.pk if user else None, 'created': '__timezone.now__' }, 'MODEL': 'Video', 'tasks': [{ 'arguments': { 'path': fpath, 'map': [{ 'arguments': { 'map': [{ 'operation': 'perform_video_decode', 'arguments': { 'segments_batch_size': defaults. DEFAULT_SEGMENTS_BATCH_SIZE, 'rate': rate, 'map': json.load( file( "../configs/custom_defaults/video_processing.json" )) } }] }, 'operation': 'perform_video_segmentation', }] }, 'video_id': '__pk__', 'operation': 'perform_import', }] }, ], } p.create_from_json(j=query, user=user) p.launch() else: raise ValueError, "Extension {} not allowed".format( filename.split('.')[-1]) return p.created_objects[0]