def parallel_user_import(domain, user_specs, upload_user):
task = parallel_user_import
total = len(user_specs)
DownloadBase.set_progress(task, 0, total)
task_list = []
for users in chunked(user_specs, USER_UPLOAD_CHUNK_SIZE):
upload_record = UserUploadRecord(domain=domain,
user_id=upload_user.user_id)
upload_record.save()
subtask = parallel_import_task.delay(domain, list(users), [],
upload_user, upload_record.pk)
task_list.append(subtask)
incomplete = True
while incomplete:
subtask_progress = 0
incomplete = False
for subtask in task_list:
try:
subtask.get(timeout=1, disable_sync_subtasks=False)
except TimeoutError:
incomplete = True
subtask_progress += get_task_progress(subtask).current or 0
else:
# The task is done, just count the rows in the result
subtask_progress += len(subtask.result['messages']['rows'])
DownloadBase.set_progress(task, subtask_progress, total)
# all tasks are done, collect results
rows = []
errors = []
for subtask in task_list:
rows.extend(subtask.result['messages']['rows'])
errors.extend(subtask.result['messages']['errors'])
messages = {'rows': rows, 'errors': errors}
return {'messages': messages}
def test_user_upload_record(self):
upload_record = UserUploadRecord(domain=self.domain, user_id='5')
upload_record.save()
task_result = import_users_and_groups.si(self.domain.name, [self.spec],
[], self.uploading_user,
upload_record.pk).apply()
rows = task_result.result
upload_record.refresh_from_db()
self.assertEqual(rows['messages'], upload_record.result)