def get_progress(self):
task_progress = get_task_progress(self.task)
return {
'current': task_progress.current,
'total': task_progress.total,
'percent': task_progress.percent,
'error': task_progress.error,
'error_message': task_progress.error_message,
}
def get_progress(self):
task_progress = get_task_progress(self.task)
return {
'current': task_progress.current,
'total': task_progress.total,
'percent': task_progress.percent,
'error': task_progress.error,
'error_message': task_progress.error_message,
}
def parallel_user_import(domain, user_specs, upload_user):
task = parallel_user_import
total = len(user_specs)
DownloadBase.set_progress(task, 0, total)
task_list = []
for users in chunked(user_specs, USER_UPLOAD_CHUNK_SIZE):
upload_record = UserUploadRecord(domain=domain,
user_id=upload_user.user_id)
upload_record.save()
subtask = parallel_import_task.delay(domain, list(users), [],
upload_user, upload_record.pk)
task_list.append(subtask)
incomplete = True
while incomplete:
subtask_progress = 0
incomplete = False
for subtask in task_list:
try:
subtask.get(timeout=1, disable_sync_subtasks=False)
except TimeoutError:
incomplete = True
subtask_progress += get_task_progress(subtask).current or 0
else:
# The task is done, just count the rows in the result
subtask_progress += len(subtask.result['messages']['rows'])
DownloadBase.set_progress(task, subtask_progress, total)
# all tasks are done, collect results
rows = []
errors = []
for subtask in task_list:
rows.extend(subtask.result['messages']['rows'])
errors.extend(subtask.result['messages']['errors'])
messages = {'rows': rows, 'errors': errors}
return {'messages': messages}