def import_vdn_dataset_url(server, url, user):
r = requests.get(url)
response = r.json()
vdn_dataset = create_dataset(response, server)
vdn_dataset.save()
video = Video()
if user:
video.uploader = user
video.name = vdn_dataset.name
video.vdn_dataset = vdn_dataset
video.save()
if vdn_dataset.download_url:
task_name = 'import_vdn_file'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.operation = task_name
import_video_task.save()
app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name])
elif vdn_dataset.aws_key and vdn_dataset.aws_bucket:
task_name = 'import_vdn_s3'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.operation = task_name
import_video_task.save()
app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name])
else:
raise NotImplementedError
def launch_task(self, t, created_pk=None):
if created_pk:
if t.get('video_id', '') == '__pk__':
t['video_id'] = created_pk
for k, v in t.get('arguments', {}).iteritems():
if v == '__pk__':
t['arguments'][k] = created_pk
if 'video_id' in t:
v = Video.objects.get(pk=t['video_id'])
map_filters = get_map_filters(t, v)
else:
map_filters = [{}]
for f in map_filters:
args = copy.deepcopy(t.get(
'arguments', {})) # make copy so that spec isnt mutated.
if f:
if 'filters' not in args:
args['filters'] = f
else:
args['filters'].update(f)
dt = TEvent()
dt.parent_process = self.process
dt.task_group_id = t['task_group_id']
if 'video_id' in t:
dt.video_id = t['video_id']
dt.arguments = args
dt.queue, op = get_queue_name_and_operation(
t['operation'], t.get('arguments', {}))
dt.operation = op
dt.save()
self.task_results[dt.pk] = app.send_task(name=dt.operation,
args=[
dt.pk,
],
queue=dt.queue)
def handle_downloaded_file(downloaded, video, name, extract=True, user=None, rate=30,
rescale=0, ):
video.name = name
video.save()
filename = downloaded.split('/')[-1]
if filename.endswith('.dva_export.zip'):
create_video_folders(video, create_subdirs=False)
os.rename(downloaded, '{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]))
video.uploaded = True
video.save()
task_name = 'import_video_by_id'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.save()
app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name])
elif filename.endswith('.mp4') or filename.endswith('.flv') or filename.endswith('.zip'):
create_video_folders(video, create_subdirs=True)
os.rename(downloaded,
'{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]))
video.uploaded = True
if filename.endswith('.zip'):
video.dataset = True
video.save()
if extract:
extract_frames_task = TEvent()
extract_frames_task.arguments_json = json.dumps({'rate': rate, 'rescale': rescale})
extract_frames_task.video = video
task_name = 'extract_frames' if video.dataset else 'segment_video'
extract_frames_task.operation = task_name
extract_frames_task.save()
app.send_task(name=task_name, args=[extract_frames_task.pk, ],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
else:
raise ValueError, "Extension {} not allowed".format(filename.split('.')[-1])
return video
def handle_downloaded_file(downloaded, video, name, extract=True, user=None, rate=30, rescale=0):
video.name = name
video.save()
filename = downloaded.split('/')[-1]
if filename.endswith('.dva_export.zip'):
create_video_folders(video, create_subdirs=False)
os.rename(downloaded, '{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]))
video.uploaded = True
video.save()
operation = 'import_video_by_id'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.save()
app.send_task(name=operation, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[operation])
elif filename.endswith('.mp4') or filename.endswith('.flv') or filename.endswith('.zip'):
create_video_folders(video, create_subdirs=True)
os.rename(downloaded,
'{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]))
video.uploaded = True
if filename.endswith('.zip'):
video.dataset = True
video.save()
if extract:
p = processing.DVAPQLProcess()
if video.dataset:
query = {
'process_type':DVAPQL.PROCESS,
'tasks':[
{
'arguments':{'rate': rate, 'rescale': rescale,'next_tasks':settings.DEFAULT_PROCESSING_PLAN},
'video_id':video.pk,
'operation': 'extract_frames',
}
]
}
else:
query = {
'process_type':DVAPQL.PROCESS,
'tasks':[
{
'arguments':{'next_tasks':[
{'operation':'decode_video',
'arguments':{
'rate': rate,
'rescale': rescale,
'next_tasks':settings.DEFAULT_PROCESSING_PLAN
}
}
]},
'video_id':video.pk,
'operation': 'segment_video',
}
]
}
p.create_from_json(j=query,user=user)
p.launch()
else:
raise ValueError, "Extension {} not allowed".format(filename.split('.')[-1])
return video
def handle_uploaded_file(f, name, extract=True, user=None, rate=30, rescale=0):
video = Video()
if user:
video.uploader = user
video.name = name
video.save()
primary_key = video.pk
filename = f.name
filename = filename.lower()
if filename.endswith('.dva_export.zip'):
create_video_folders(video, create_subdirs=False)
with open('{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]),
'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
video.uploaded = True
video.save()
task_name = 'import_video_by_id'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.save()
app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name])
elif filename.endswith('.mp4') or filename.endswith('.flv') or filename.endswith('.zip'):
create_video_folders(video, create_subdirs=True)
with open('{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]),
'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
video.uploaded = True
if filename.endswith('.zip'):
video.dataset = True
video.save()
if extract:
p = processing.DVAPQLProcess()
query = {
'process_type':DVAPQL.PROCESS,
'tasks':[
{
'arguments_json':{'rate': rate, 'rescale': rescale},
'video_id':video.pk,
'operation': 'extract_frames' if video.dataset else 'segment_video',
}
]
}
p.create_from_json(j=query,user=user)
p.launch()
else:
raise ValueError, "Extension {} not allowed".format(filename.split('.')[-1])
return video
def handle_youtube_video(name,
url,
extract=True,
user=None,
perform_scene_detection=True,
rate=30,
rescale=0):
video = Video()
if user:
video.uploader = user
video.name = name
video.url = url
video.youtube_video = True
video.save()
task_name = 'extract_frames_by_id'
extract_frames_task = TEvent()
extract_frames_task.video = video
extract_frames_task.operation = task_name
extract_frames_task.arguments_json = json.dumps({
'perform_scene_detection': perform_scene_detection,
'rate': rate,
'rescale': rescale
})
extract_frames_task.save()
if extract:
app.send_task(name=task_name,
args=[
extract_frames_task.pk,
],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
return video
def delete_video_object(video_pk,deleter,garbage_collection=True):
video = Video.objects.get(pk=video_pk)
deleted = DeletedVideo()
deleted.name = video.name
deleted.deleter = deleter
deleted.uploader = video.uploader
deleted.url = video.url
deleted.description = video.description
deleted.original_pk = video_pk
deleted.save()
video.delete()
if garbage_collection:
delete_task = TEvent()
delete_task.arguments_json = json.dumps({'video_pk': video_pk})
delete_task.operation = 'delete_video_by_id'
delete_task.save()
queue = settings.TASK_NAMES_TO_QUEUE[delete_task.operation]
_ = app.send_task(name=delete_task.operation, args=[delete_task.pk], queue=queue)
def import_vdn_dataset_url(server,url,user):
r = requests.get(url)
response = r.json()
vdn_dataset = create_dataset(response, server)
vdn_dataset.save()
video = Video()
if user:
video.uploader = user
video.name = vdn_dataset.name
video.vdn_dataset = vdn_dataset
video.save()
primary_key = video.pk
create_video_folders(video, create_subdirs=False)
task_name = 'import_video_by_id'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.save()
app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name])
def handle_uploaded_file(f, name, extract=True, user=None, rate=30, rescale=0):
video = Video()
if user:
video.uploader = user
video.name = name
video.save()
primary_key = video.pk
filename = f.name
filename = filename.lower()
if filename.endswith('.dva_export.zip'):
create_video_folders(video, create_subdirs=False)
with open('{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]),
'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
video.uploaded = True
video.save()
task_name = 'import_video_by_id'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.save()
app.send_task(name=task_name, args=[import_video_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name])
elif filename.endswith('.mp4') or filename.endswith('.flv') or filename.endswith('.zip'):
create_video_folders(video, create_subdirs=True)
with open('{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk, filename.split('.')[-1]),
'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
video.uploaded = True
if filename.endswith('.zip'):
video.dataset = True
video.save()
if extract:
extract_frames_task = TEvent()
extract_frames_task.arguments_json = json.dumps({'rate': rate,'rescale': rescale})
extract_frames_task.video = video
task_name = 'extract_frames' if video.dataset else 'segment_video'
extract_frames_task.operation = task_name
extract_frames_task.save()
app.send_task(name=task_name, args=[extract_frames_task.pk, ],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
else:
raise ValueError, "Extension {} not allowed".format(filename.split('.')[-1])
return video
def import_vdn_detector_url(server, url, user):
r = requests.get(url)
response = r.json()
vdn_detector = create_vdn_detector(response, server)
detector = CustomDetector()
detector.name = vdn_detector.name
detector.vdn_detector = vdn_detector
detector.save()
if vdn_detector.download_url:
task_name = 'import_vdn_detector_file'
import_vdn_detector_task = TEvent()
import_vdn_detector_task.operation = task_name
import_vdn_detector_task.arguments_json = json.dumps({'detector_pk': detector.pk})
import_vdn_detector_task.save()
app.send_task(name=task_name, args=[import_vdn_detector_task.pk, ],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
elif vdn_detector.aws_key and vdn_detector.aws_bucket:
raise NotImplementedError
else:
raise NotImplementedError
def handle_youtube_video(name, url, extract=True, user=None, rate=30, rescale=0):
video = Video()
if user:
video.uploader = user
video.name = name
video.url = url
video.youtube_video = True
video.save()
task_name = 'segment_video'
extract_frames_task = TEvent()
extract_frames_task.video = video
extract_frames_task.operation = task_name
extract_frames_task.arguments_json = json.dumps({'rate': rate,'rescale': rescale})
extract_frames_task.save()
if extract:
app.send_task(name=task_name, args=[extract_frames_task.pk, ], queue=settings.TASK_NAMES_TO_QUEUE[task_name])
return video
def delete_video_object(video_pk,deleter,garbage_collection=True):
video = Video.objects.get(pk=video_pk)
deleted = DeletedVideo()
deleted.name = video.name
deleted.deleter = deleter
deleted.uploader = video.uploader
deleted.url = video.url
deleted.description = video.description
deleted.original_pk = video_pk
deleted.save()
video.delete()
if garbage_collection:
delete_task = TEvent()
delete_task.arguments = {'video_pk': video_pk}
delete_task.operation = 'delete_video_by_id'
delete_task.save()
queue = settings.TASK_NAMES_TO_QUEUE[delete_task.operation]
_ = app.send_task(name=delete_task.operation, args=[delete_task.pk], queue=queue)
def import_vdn_detector_url(server, url, user):
r = requests.get(url)
response = r.json()
vdn_detector = create_vdn_detector(response, server)
detector = CustomDetector()
detector.name = vdn_detector.name
detector.vdn_detector = vdn_detector
detector.save()
if vdn_detector.download_url:
task_name = 'import_vdn_detector_file'
import_vdn_detector_task = TEvent()
import_vdn_detector_task.operation = task_name
import_vdn_detector_task.arguments_json = json.dumps({'detector_pk': detector.pk})
import_vdn_detector_task.save()
app.send_task(name=task_name, args=[import_vdn_detector_task.pk, ],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
elif vdn_detector.aws_key and vdn_detector.aws_bucket:
raise NotImplementedError
else:
raise NotImplementedError
def create_event(e, v):
de = TEvent()
de.imported = True
de.started = e.get('started', False)
de.start_ts = e.get('start_ts', None)
de.completed = e.get('completed', False)
de.errored = e.get('errored', False)
de.error_message = e.get('error_message', "")
de.video_id = v.pk
de.operation = e.get('operation', "")
de.created = e['created']
if 'seconds' in e:
de.duration = e.get('seconds', -1)
else:
de.duration = e.get('duration', -1)
de.arguments = e.get('arguments', {})
de.task_id = e.get('task_id', "")
return de
def launch_task(self, t):
for k, v in t.get('arguments', {}).iteritems():
if (type(v) is str or type(v) is unicode) and v.startswith('__created__'):
t['arguments'][k] = self.get_created_object_pk(v)
if 'video_id' in t or 'video_selector' in t:
if (type(t['video_id']) is str or type(t['video_id']) is unicode) and t['video_id'].startswith(
'__created__'):
t['video_id'] = self.get_created_object_pk(t['video_id'])
v = Video.objects.get(pk=t['video_id'])
elif 'video_selector' in t:
v = Video.objects.get(**t['video_selector'])
t['video_id'] = v.pk
else:
v = Video.objects.get(pk=t['video_id'])
map_filters = get_map_filters(t, v)
else:
map_filters = [{}]
# This is useful in case of perform_stream_capture where batch size is used but number of segments is unknown
if map_filters == []:
map_filters = [{}]
for f in map_filters:
args = copy.deepcopy(t.get('arguments', {})) # make copy so that spec isnt mutated.
if f:
if 'filters' not in args:
args['filters'] = f
else:
args['filters'].update(f)
dt = TEvent()
dt.parent_process = self.process
dt.task_group_id = t['task_group_id']
dt.parent = self.root_task
if 'video_id' in t:
dt.video_id = t['video_id']
if 'training_set_id' in t:
dt.training_set_id = t['training_set_id']
elif 'training_set_selector' in t:
dt.training_set_id = TrainingSet.objects.get(**t['training_set_selector'])
dt.arguments = args
dt.queue, op = get_queue_name_and_operation(t['operation'], t.get('arguments', {}))
dt.operation = op
dt.save()
self.task_results[dt.pk] = app.send_task(name=dt.operation, args=[dt.pk, ], queue=dt.queue)
def import_vdn_dataset_url(server, url, user):
r = requests.get(url)
response = r.json()
vdn_dataset = create_dataset(response, server)
vdn_dataset.save()
video = Video()
if user:
video.uploader = user
video.name = vdn_dataset.name
video.vdn_dataset = vdn_dataset
video.save()
if vdn_dataset.download_url:
task_name = 'import_vdn_file'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.operation = task_name
import_video_task.save()
app.send_task(name=task_name,
args=[
import_video_task.pk,
],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
elif vdn_dataset.aws_key and vdn_dataset.aws_bucket:
task_name = 'import_vdn_s3'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.operation = task_name
import_video_task.save()
app.send_task(name=task_name,
args=[
import_video_task.pk,
],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
else:
raise NotImplementedError
def handle_uploaded_file(f,
name,
extract=True,
user=None,
perform_scene_detection=True,
rate=30,
rescale=0):
video = Video()
if user:
video.uploader = user
video.name = name
video.save()
primary_key = video.pk
filename = f.name
filename = filename.lower()
if filename.endswith('.dva_export.zip'):
create_video_folders(video, create_subdirs=False)
with open(
'{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk, video.pk,
filename.split('.')[-1]),
'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
video.uploaded = True
video.save()
task_name = 'import_video_by_id'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.save()
app.send_task(name=task_name,
args=[
import_video_task.pk,
],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
elif filename.endswith('.mp4') or filename.endswith(
'.flv') or filename.endswith('.zip'):
create_video_folders(video, create_subdirs=True)
with open(
'{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT, video.pk,
video.pk,
filename.split('.')[-1]),
'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
video.uploaded = True
if filename.endswith('.zip'):
video.dataset = True
video.save()
if extract:
extract_frames_task = TEvent()
extract_frames_task.arguments_json = json.dumps({
'perform_scene_detection':
perform_scene_detection,
'rate':
rate,
'rescale':
rescale
})
extract_frames_task.video = video
task_name = 'extract_frames_by_id'
extract_frames_task.operation = task_name
extract_frames_task.save()
app.send_task(name=task_name,
args=[
extract_frames_task.pk,
],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
else:
raise ValueError, "Extension {} not allowed".format(
filename.split('.')[-1])
return video
def handle_downloaded_file(
downloaded,
video,
name,
extract=True,
user=None,
perform_scene_detection=True,
rate=30,
rescale=0,
):
video.name = name
video.save()
filename = downloaded.split('/')[-1]
if filename.endswith('.dva_export.zip'):
create_video_folders(video, create_subdirs=False)
os.rename(
downloaded, '{}/{}/{}.{}'.format(settings.MEDIA_ROOT, video.pk,
video.pk,
filename.split('.')[-1]))
video.uploaded = True
video.save()
task_name = 'import_video_by_id'
import_video_task = TEvent()
import_video_task.video = video
import_video_task.save()
app.send_task(name=task_name,
args=[
import_video_task.pk,
],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
elif filename.endswith('.mp4') or filename.endswith(
'.flv') or filename.endswith('.zip'):
create_video_folders(video, create_subdirs=True)
os.rename(
downloaded, '{}/{}/video/{}.{}'.format(settings.MEDIA_ROOT,
video.pk, video.pk,
filename.split('.')[-1]))
video.uploaded = True
if filename.endswith('.zip'):
video.dataset = True
video.save()
if extract:
extract_frames_task = TEvent()
extract_frames_task.arguments_json = json.dumps({
'perform_scene_detection':
perform_scene_detection,
'rate':
rate,
'rescale':
rescale
})
extract_frames_task.video = video
task_name = 'extract_frames_by_id'
extract_frames_task.operation = task_name
extract_frames_task.save()
app.send_task(name=task_name,
args=[
extract_frames_task.pk,
],
queue=settings.TASK_NAMES_TO_QUEUE[task_name])
else:
raise ValueError, "Extension {} not allowed".format(
filename.split('.')[-1])
return video
def create_event(e, v):
de = TEvent()
de.event_type = e.get('event_type', de.VIDEO)
de.started = e.get('started', False)
de.completed = e.get('completed', False)
de.errored = e.get('errored', False)
de.error_message = e.get('error_message', "")
de.video_id = v.pk
de.operation = e.get('operation', "")
de.created = e['created']
de.seconds = e.get('seconds', -1)
de.file_name = e.get('file_name', "")
de.key = e.get('key', "")
de.bucket = e.get('bucket', "")
de.requester_pays = e.get('requester_pays', False)
de.clustering = e.get('clustering', None)
de.arguments = e.get('arguments', {})
de.task_id = e.get('task_id', "")
return de
def create_event(e,v):
de = TEvent()
de.event_type = e.get('event_type',de.VIDEO)
de.started = e.get('started',False)
de.completed = e.get('completed',False)
de.errored = e.get('errored',False)
de.error_message = e.get('error_message',"")
de.video_id = v.pk
de.operation = e.get('operation',"")
de.created = e['created']
de.seconds = e.get('seconds',-1)
de.file_name = e.get('file_name',"")
de.key = e.get('key',"")
de.bucket = e.get('bucket',"")
de.requester_pays = e.get('requester_pays',False)
de.clustering = e.get('clustering',None)
de.arguments_json = e.get('arguments_json',"")
de.task_id = e.get('task_id',"")
return de
def create_event(e, v, dt):
de = TEvent()
de.imported = dt
de.id = e['id'] # id is a uuid
de.results = e.get('results', None)
de.started = e.get('started', False)
de.start_ts = e.get('start_ts', None)
# Completed is set to False since we it will be completed only when task performing import uploads all the data.
de.completed = False
de.errored = e.get('errored', False)
de.error_message = e.get('error_message', "")
de.video_id = v.pk
de.operation = e.get('operation', "")
de.created = e['created']
de.min_frame_index = e.get('min_frame_index', None)
de.max_frame_index = e.get('max_frame_index', None)
de.duration = e.get('duration', -1)
de.arguments = e.get('arguments', {})
de.task_id = e.get('task_id', "")
return de
