def import_vdn_dataset_url(server, url, user, cached_response):
response = None
try:
r = requests.get(url)
response = r.json()
except:
pass
if not response:
response = cached_response
video = Video()
video.description = "import from {} : {} ".format(server.url,
response['description'])
if user:
video.uploader = user
video.name = response['name']
video.save()
if response['download_url']:
p = processing.DVAPQLProcess()
query = {
'process_type':
DVAPQL.PROCESS,
'tasks': [{
'arguments': {
'source': 'VDN_URL',
'url': response['download_url']
},
'video_id': video.pk,
'operation': 'perform_import',
}]
}
p.create_from_json(j=query, user=user)
p.launch()
elif response['aws_key'] and response['aws_bucket']:
p = processing.DVAPQLProcess()
query = {
'process_type':
DVAPQL.PROCESS,
'tasks': [{
'arguments': {
'source': 'VDN_S3',
'key': response['aws_key'],
'bucket': response['aws_bucket']
},
'video_id': video.pk,
'operation': 'perform_import',
}]
}
p.create_from_json(j=query, user=user)
p.launch()
else:
raise NotImplementedError
def import_vdn_detector_url(server, url, user, cached_response):
response = None
try:
r = requests.get(url)
response = r.json()
except:
pass
if not response:
response = cached_response
detector = DeepModel()
detector.model_type = DeepModel.DETECTOR
detector.name = response['name']
detector.detector_type = response.get('detector_type', DeepModel.YOLO)
detector.save()
if response.get('download_url', False):
p = processing.DVAPQLProcess()
query = {
'process_type':
DVAPQL.PROCESS,
'tasks': [{
'arguments': {
'detector_pk': detector.pk,
'download_url': response['download_url']
},
'operation': 'perform_detector_import',
}]
}
p.create_from_json(j=query, user=user)
p.launch()
elif response.get('aws_key', False) and response.get('aws_bucket', False):
raise NotImplementedError
else:
raise NotImplementedError
def import_vdn_detector_url(server, url, user, cached_response):
response = None
try:
r = requests.get(url)
response = r.json()
except:
pass
if not response:
response = cached_response
p = processing.DVAPQLProcess()
query = {
'process_type':
DVAPQL.PROCESS,
'create': [{
'MODEL':
'TrainedModel',
'spec': {
'name': response['name'],
'detector_type': TrainedModel.DETECTOR
},
'tasks': [
{
'operation': 'perform_detector_import',
'arguments': {
'path': response['path'],
'detector_pk': '__pk__'
},
},
]
}]
}
p.create_from_json(j=query, user=user)
p.launch()
def qt_lopq():
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from dvaapp.models import Retriever, DeepModel,TEvent,DVAPQL
from dvaapp import processing
spec = {
'process_type':DVAPQL.PROCESS,
'create':[{
'MODEL':'Retriever',
'spec':{
'algorithm':Retriever.LOPQ,
'arguments':{'components': 32, 'm': 8, 'v': 8, 'sub': 128},
'source_filters':{'indexer_shasum': DeepModel.objects.get(name="inception",model_type=DeepModel.INDEXER).shasum}
},
'tasks':[
{
'operation':'perform_retriever_creation',
'arguments':{'retriever_pk': '__pk__'}
}
]
},]
}
p = processing.DVAPQLProcess()
p.create_from_json(j=spec,user=None)
p.launch()
p.wait()
def import_vdn_dataset_url(server, url, user, cached_response):
response = None
try:
r = requests.get(url)
response = r.json()
except:
pass
if not response:
response = cached_response
p = processing.DVAPQLProcess()
query = {
'process_type': DVAPQL.PROCESS,
'create': [
{
'spec': {
'name': response['name'],
'uploader_id': user.pk if user else None,
'url': response['path'],
'created': '__timezone.now__',
'description':"import from {} : {} ".format(server.url,response['description'])
},
'MODEL': 'Video',
'tasks': [
{
'arguments': {'path':response['path']},
'video_id': '__pk__',
'operation': 'perform_import',
}
]
},
],
}
p.create_from_json(j=query, user=user)
p.launch()
def delete_video_object(video_pk, deleter):
p = processing.DVAPQLProcess()
query = {
'process_type': dvaapp.models.DVAPQL.PROCESS,
'delete': [{
'MODEL': 'Video',
'selector': {
'pk': video_pk
},
}]
}
p.create_from_json(j=query, user=deleter)
p.launch()
def delete_video_object(video_pk,deleter):
p = processing.DVAPQLProcess()
query = {
'process_type': DVAPQL.PROCESS,
'tasks': [
{
'arguments': {'video_pk': video_pk,'deleter_pk':deleter.pk},
'operation': 'perform_deletion',
}
]
}
p.create_from_json(j=query, user=deleter)
p.launch()
def handle_uploaded_file(f,
name,
extract=True,
user=None,
rate=None,
rescale=None):
if rate is None:
rate = defaults.DEFAULT_RATE
if rescale is None:
rescale = defaults.DEFAULT_RESCALE
video = Video()
if user:
video.uploader = user
video.name = name
video.save()
filename = f.name
filename = filename.lower()
if filename.endswith('.dva_export.zip'):
video.create_directory(create_subdirs=False)
with open(
'{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk,
filename.split('.')[-1]),
'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
if settings.DISABLE_NFS or settings.HEROKU_DEPLOY:
fpath = '/{}/{}.{}'.format(video.pk, video.pk,
filename.split('.')[-1])
fs.upload_file_to_remote(fpath)
os.remove('{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk,
video.pk,
filename.split('.')[-1]))
video.uploaded = True
video.save()
p = processing.DVAPQLProcess()
query = {
'process_type':
DVAPQL.PROCESS,
'tasks': [{
'arguments': {
'source': 'LOCAL'
},
'video_id': video.pk,
'operation': 'perform_import',
}]
}
p.create_from_json(j=query, user=user)
p.launch()
elif filename.endswith('.mp4') or filename.endswith(
'.flv') or filename.endswith('.zip'):
video.create_directory(create_subdirs=True)
with open(
'{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk,
video.pk,
filename.split('.')[-1]),
'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
if settings.DISABLE_NFS or settings.HEROKU_DEPLOY:
fpath = '/{}/video/{}.{}'.format(video.pk, video.pk,
filename.split('.')[-1])
fs.upload_file_to_remote(fpath)
os.remove('{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk,
video.pk,
filename.split('.')[-1]))
video.uploaded = True
if filename.endswith('.zip'):
video.dataset = True
video.save()
if extract:
p = processing.DVAPQLProcess()
if video.dataset:
query = {
'process_type':
DVAPQL.PROCESS,
'tasks': [{
'arguments': {
'rescale': rescale,
'next_tasks':
defaults.DEFAULT_PROCESSING_PLAN_DATASET
},
'video_id': video.pk,
'operation': 'perform_dataset_extraction',
}]
}
else:
query = {
'process_type':
DVAPQL.PROCESS,
'tasks': [{
'arguments': {
'next_tasks': [{
'operation': 'perform_video_decode',
'arguments': {
'segments_batch_size':
defaults.DEFAULT_SEGMENTS_BATCH_SIZE,
'rate':
rate,
'rescale':
rescale,
'next_tasks':
defaults.DEFAULT_PROCESSING_PLAN_VIDEO
}
}]
},
'video_id': video.pk,
'operation': 'perform_video_segmentation',
}]
}
p.create_from_json(j=query, user=user)
p.launch()
else:
raise ValueError, "Extension {} not allowed".format(
filename.split('.')[-1])
return video
def handle_uploaded_file(f, name, user=None, rate=None):
if rate is None:
rate = defaults.DEFAULT_RATE
filename = f.name
filename = filename.lower()
vuid = str(uuid.uuid1()).replace('-', '_')
extension = filename.split('.')[-1]
if filename.endswith('.dva_export.zip'):
local_fname = '{}/ingest/{}.dva_export.zip'.format(
settings.MEDIA_ROOT, vuid)
fpath = '/ingest/{}.dva_export.zip'.format(vuid)
with open(local_fname, 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
if settings.DISABLE_NFS:
fs.upload_file_to_remote(fpath)
os.remove(local_fname)
p = processing.DVAPQLProcess()
query = {
'process_type':
DVAPQL.PROCESS,
'create': [
{
'spec': {
'name': name,
'uploader_id': user.pk if user else None,
'created': '__timezone.now__'
},
'MODEL':
'Video',
'tasks': [{
'arguments': {
'path': fpath
},
'video_id': '__pk__',
'operation': 'perform_import',
}]
},
],
}
p.create_from_json(j=query, user=user)
p.launch()
elif extension in ['zip', 'gz', 'json', 'mp4']:
local_fname = '{}/ingest/{}.{}'.format(settings.MEDIA_ROOT, vuid,
filename.split('.')[-1])
fpath = '/ingest/{}.{}'.format(vuid, filename.split('.')[-1])
with open(local_fname, 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
if settings.DISABLE_NFS:
fs.upload_file_to_remote(fpath)
os.remove(local_fname)
p = processing.DVAPQLProcess()
if extension == 'zip':
query = {
'process_type':
DVAPQL.PROCESS,
'create': [
{
'spec': {
'name': name,
'dataset': True,
'uploader_id': user.pk if user else None,
'created': '__timezone.now__'
},
'MODEL':
'Video',
'tasks': [{
'arguments': {
'path':
fpath,
'map': [{
'arguments': {
'map':
json.load(
file(
"../configs/custom_defaults/dataset_processing.json"
))
},
'operation':
'perform_dataset_extraction',
}]
},
'video_id': '__pk__',
'operation': 'perform_import'
}]
},
],
}
elif extension == 'json' or extension == 'gz':
query = {
'process_type':
DVAPQL.PROCESS,
'create': [
{
'spec': {
'name': name,
'dataset': True,
'uploader_id': user.pk if user else None,
'created': '__timezone.now__'
},
'MODEL':
'Video',
'tasks': [{
'arguments': {
'path':
fpath,
'map': [{
'operation': 'perform_frame_download',
'arguments': {
'frames_batch_size':
defaults.DEFAULT_FRAMES_BATCH_SIZE,
'map':
json.load(
file(
"../configs/custom_defaults/framelist_processing.json"
))
},
}]
},
'video_id': '__pk__',
'operation': 'perform_import'
}]
},
],
}
else:
query = {
'process_type':
DVAPQL.PROCESS,
'create': [
{
'spec': {
'name': name,
'uploader_id': user.pk if user else None,
'created': '__timezone.now__'
},
'MODEL':
'Video',
'tasks': [{
'arguments': {
'path':
fpath,
'map': [{
'arguments': {
'map': [{
'operation':
'perform_video_decode',
'arguments': {
'segments_batch_size':
defaults.
DEFAULT_SEGMENTS_BATCH_SIZE,
'rate':
rate,
'map':
json.load(
file(
"../configs/custom_defaults/video_processing.json"
))
}
}]
},
'operation':
'perform_video_segmentation',
}]
},
'video_id': '__pk__',
'operation': 'perform_import',
}]
},
],
}
p.create_from_json(j=query, user=user)
p.launch()
else:
raise ValueError, "Extension {} not allowed".format(
filename.split('.')[-1])
return p.created_objects[0]
