def process_next(dt, inject_filters=None, custom_next_tasks=None, sync=True, launch_next=True, map_filters=None):
if custom_next_tasks is None:
custom_next_tasks = []
task_id = dt.pk
launched = []
args = copy.deepcopy(dt.arguments)
logging.info("next tasks for {}".format(dt.operation))
next_tasks = args.get('map', []) if args and launch_next else []
if sync and settings.MEDIA_BUCKET:
for k in SYNC_TASKS.get(dt.operation, []):
if settings.ENABLE_CLOUDFS:
dirname = k['arguments'].get('dirname', None)
task_shared.upload(dirname, task_id, dt.video_id)
else:
launched += launch_tasks(k, dt, inject_filters, None, 'sync')
for k in next_tasks + custom_next_tasks:
if map_filters is None:
map_filters = get_map_filters(k, dt.video)
launched += launch_tasks(k, dt, inject_filters, map_filters, 'map')
for reduce_task in args.get('reduce', []):
next_task = TEvent.objects.create(video=dt.video, operation="perform_reduce",
arguments=reduce_task['arguments'], parent=dt,
task_group_id=reduce_task['task_group_id'],
parent_process_id=dt.parent_process_id, queue=settings.Q_REDUCER)
launched.append(app.send_task(next_task.operation, args=[next_task.pk, ], queue=settings.Q_REDUCER).id)
return launched
def process_next(task_id,inject_filters=None,custom_next_tasks=None,sync=True,launch_next=True):
if custom_next_tasks is None:
custom_next_tasks = []
dt = TEvent.objects.get(pk=task_id)
launched = []
logging.info("next tasks for {}".format(dt.operation))
next_tasks = dt.arguments.get('next_tasks',[]) if dt.arguments and launch_next else []
if sync and settings.MEDIA_BUCKET:
for k in SYNC_TASKS.get(dt.operation,[]):
if settings.DISABLE_NFS:
dirname = k['arguments'].get('dirname',None)
task_shared.upload(dirname,task_id,dt.video_id)
else:
launched += launch_tasks(k,dt,inject_filters,None,'sync')
for k in next_tasks+custom_next_tasks:
map_filters = get_map_filters(k,dt.video)
launched += launch_tasks(k, dt, inject_filters,map_filters,'next_tasks')
return launched