def launch_task(self, t, created_pk=None):
if created_pk:
if t.get('video_id', '') == '__pk__':
t['video_id'] = created_pk
for k, v in t.get('arguments', {}).iteritems():
if v == '__pk__':
t['arguments'][k] = created_pk
if 'video_id' in t:
v = Video.objects.get(pk=t['video_id'])
map_filters = get_map_filters(t, v)
else:
map_filters = [{}]
for f in map_filters:
args = copy.deepcopy(t.get(
'arguments', {})) # make copy so that spec isnt mutated.
if f:
if 'filters' not in args:
args['filters'] = f
else:
args['filters'].update(f)
dt = TEvent()
dt.parent_process = self.process
if 'video_id' in t:
dt.video_id = t['video_id']
dt.arguments = args
dt.queue, op = get_queue_name_and_operation(
t['operation'], t.get('arguments', {}))
dt.operation = op
dt.save()
self.task_results[dt.pk] = app.send_task(name=dt.operation,
args=[
dt.pk,
],
queue=dt.queue)
def launch_task(self, t):
for k, v in t.get('arguments', {}).iteritems():
if (type(v) is str
or type(v) is unicode) and v.startswith('__created__'):
t['arguments'][k] = self.get_created_object_pk(v)
dv = None
if t['operation'] in settings.NON_PROCESSING_TASKS:
dv = None
elif 'video_id' in t:
if t['video_id'].startswith('__created__'):
t['video_id'] = self.get_created_object_pk(t['video_id'])
dv = Video.objects.get(pk=t['video_id'])
elif 'video_selector' in t['arguments']:
dv = Video.objects.get(**t['arguments']['video_selector'])
t['video_id'] = dv.pk
if dv:
map_filters = get_map_filters(t, dv)
else:
map_filters = [{}]
# This is useful in case of perform_stream_capture where batch size is used but number of segments is unknown
if map_filters == []:
map_filters = [{}]
for f in map_filters:
args = copy.deepcopy(t.get(
'arguments', {})) # make copy so that spec isnt mutated.
if f:
if 'filters' not in args:
args['filters'] = f
else:
args['filters'].update(f)
dt = TEvent()
dt.parent_process = self.process
dt.task_group_id = t['task_group_id']
dt.parent = self.root_task
if 'video_id' in t:
dt.video_id = t['video_id']
if 'training_set_id' in t:
dt.training_set_id = t['training_set_id']
elif 'trainingset_selector' in t['arguments']:
dt.training_set_id = TrainingSet.objects.get(
**t['arguments']['trainingset_selector'])
dt.arguments = args
dt.queue, op = get_queue_name_and_operation(
t['operation'], t.get('arguments', {}))
dt.operation = op
dt.save()
self.task_results[dt.pk] = app.send_task(name=dt.operation,
args=[
dt.pk,
],
queue=dt.queue)
def create_event(e,v):
de = TEvent()
de.started = e.get('started',False)
de.start_ts = e.get('start_ts',None)
de.completed = e.get('completed',False)
de.errored = e.get('errored',False)
de.error_message = e.get('error_message',"")
de.video_id = v.pk
de.operation = e.get('operation',"")
de.created = e['created']
de.seconds = e.get('seconds',-1)
de.arguments = e.get('arguments',{})
de.task_id = e.get('task_id',"")
return de
def delete_video_object(video_pk,deleter,garbage_collection=True):
video = Video.objects.get(pk=video_pk)
deleted = DeletedVideo()
deleted.name = video.name
deleted.deleter = deleter
deleted.uploader = video.uploader
deleted.url = video.url
deleted.description = video.description
deleted.original_pk = video_pk
deleted.save()
video.delete()
if garbage_collection:
delete_task = TEvent()
delete_task.arguments = {'video_pk': video_pk}
delete_task.operation = 'delete_video_by_id'
delete_task.save()
queue = settings.TASK_NAMES_TO_QUEUE[delete_task.operation]
_ = app.send_task(name=delete_task.operation, args=[delete_task.pk], queue=queue)
def create_event(e, v):
de = TEvent()
de.event_type = e.get('event_type', de.VIDEO)
de.started = e.get('started', False)
de.completed = e.get('completed', False)
de.errored = e.get('errored', False)
de.error_message = e.get('error_message', "")
de.video_id = v.pk
de.operation = e.get('operation', "")
de.created = e['created']
de.seconds = e.get('seconds', -1)
de.file_name = e.get('file_name', "")
de.key = e.get('key', "")
de.bucket = e.get('bucket', "")
de.requester_pays = e.get('requester_pays', False)
de.clustering = e.get('clustering', None)
de.arguments = e.get('arguments', {})
de.task_id = e.get('task_id', "")
return de
def import_vdn_detector_url(server, url, user):
r = requests.get(url)
response = r.json()
vdn_detector = create_vdn_detector(response, server)
detector = CustomDetector()
detector.name = vdn_detector.name
detector.vdn_detector = vdn_detector
detector.save()
if vdn_detector.download_url:
operation = 'import_vdn_detector_file'
import_vdn_detector_task = TEvent()
import_vdn_detector_task.operation = operation
import_vdn_detector_task.arguments = {'detector_pk': detector.pk}
import_vdn_detector_task.save()
app.send_task(name=operation, args=[import_vdn_detector_task.pk, ],
queue=settings.TASK_NAMES_TO_QUEUE[operation])
elif vdn_detector.aws_key and vdn_detector.aws_bucket:
raise NotImplementedError
else:
raise NotImplementedError
def create_event(e, v, dt):
de = TEvent()
de.imported = dt
de.id = e['id'] # id is a uuid
de.results = e.get('results', None)
de.started = e.get('started', False)
de.start_ts = e.get('start_ts', None)
de.completed = e.get('completed', False)
de.errored = e.get('errored', False)
de.error_message = e.get('error_message', "")
de.video_id = v.pk
de.operation = e.get('operation', "")
de.created = e['created']
if 'seconds' in e:
de.duration = e.get('seconds', -1)
else:
de.duration = e.get('duration', -1)
de.arguments = e.get('arguments', {})
de.task_id = e.get('task_id', "")
return de
def create_event(e, v, dt):
de = TEvent()
de.imported = dt
de.id = e['id'] # id is a uuid
de.results = e.get('results', None)
de.started = e.get('started', False)
de.start_ts = e.get('start_ts', None)
# Completed is set to False since we it will be completed only when task performing import uploads all the data.
de.completed = False
de.errored = e.get('errored', False)
de.error_message = e.get('error_message', "")
de.video_id = v.pk
de.operation = e.get('operation', "")
de.created = e['created']
de.min_frame_index = e.get('min_frame_index', None)
de.max_frame_index = e.get('max_frame_index', None)
de.duration = e.get('duration', -1)
de.arguments = e.get('arguments', {})
de.task_id = e.get('task_id', "")
return de
def launch_task(self, t, created_pk=None):
if created_pk:
if t.get('video_id', '') == '__pk__':
t['video_id'] = created_pk
for k, v in t.get('arguments', {}).iteritems():
if v == '__pk__':
t['arguments'][k] = created_pk
if 'video_id' in t:
v = Video.objects.get(pk=t['video_id'])
map_filters = get_map_filters(t, v)
else:
map_filters = [{}]
# This is useful in case of perform_stream_capture where batch size is used but number of segments is unknown
if map_filters == []:
map_filters = [{}]
for f in map_filters:
args = copy.deepcopy(t.get(
'arguments', {})) # make copy so that spec isnt mutated.
if f:
if 'filters' not in args:
args['filters'] = f
else:
args['filters'].update(f)
dt = TEvent()
dt.parent_process = self.process
dt.task_group_id = t['task_group_id']
if 'video_id' in t:
dt.video_id = t['video_id']
dt.arguments = args
dt.queue, op = get_queue_name_and_operation(
t['operation'], t.get('arguments', {}))
dt.operation = op
dt.save()
self.task_results[dt.pk] = app.send_task(name=dt.operation,
args=[
dt.pk,
],
queue=dt.queue)
