def __init__(self, settings):
"""
Setup to talk to the data service based on settings.
:param settings: UploadSettings: settings to use for uploading.
"""
self.runner = TaskRunner(TaskExecutor(settings.config.upload_workers))
self.settings = settings
self.small_item_task_builder = SmallItemUploadTaskBuilder(
self.settings, self.runner)
self.small_items = []
self.large_items = []
def test_two_adds_in_order(self):
add_command = AddCommand(10, 30)
add_command2 = AddCommand(4, 1)
executor = TaskExecutor(10)
runner = TaskRunner(executor)
runner.add(None, add_command)
runner.add(1, add_command2)
runner.run()
self.assertEqual(add_command.parent_task_result, None)
self.assertEqual(add_command.result, 40)
self.assertEqual(add_command2.parent_task_result, 40)
self.assertEqual(add_command2.result, 5)
def test_single_add(self):
add_command = AddCommand(10, 30)
executor = TaskExecutor(10)
runner = TaskRunner(executor)
runner.add(None, add_command)
runner.run()
self.assertEqual(add_command.parent_task_result, None)
self.assertEqual(add_command.result, 40)
def test_command_with_message(self):
add_command = AddCommand(10, 30)
add_command.send_message = 'ok'
add_command2 = AddCommand(4, 1)
add_command2.send_message = 'waiting'
executor = TaskExecutor(10)
runner = TaskRunner(executor)
runner.add(
None,
add_command,
)
runner.add(None, add_command2)
runner.run()
self.assertEqual(add_command.parent_task_result, None)
self.assertEqual(add_command.result, 40)
self.assertEqual(add_command2.parent_task_result, None)
self.assertEqual(add_command2.result, 5)
self.assertEqual(add_command.on_message_data, ['ok'])
self.assertEqual(add_command2.on_message_data, ['waiting'])
class ProjectUploader(object):
"""
Uploads a project based on UploadSettings.
"""
def __init__(self, settings):
"""
Setup to talk to the data service based on settings.
:param settings: UploadSettings: settings to use for uploading.
"""
self.runner = TaskRunner(TaskExecutor(settings.config.upload_workers))
self.settings = settings
self.small_item_task_builder = SmallItemUploadTaskBuilder(
self.settings, self.runner)
self.small_items = []
self.large_items = []
def run(self, local_project):
"""
Upload a project by uploading project, folders, and small files then uploading the large files.
:param local_project: LocalProject: project to upload
"""
# Walk project adding small items to runner saving large items to large_items
ProjectWalker.walk_project(local_project, self)
# Run small items in parallel
self.runner.run()
# Run parts of each large item in parallel
self.upload_large_items()
# Methods called by ProjectWalker.walk_project
def visit_project(self, item):
"""
Add create project to small task list.
"""
self.small_item_task_builder.visit_project(item)
def visit_folder(self, item, parent):
"""
Add create folder to small task list.
"""
self.small_item_task_builder.visit_folder(item, parent)
def visit_file(self, item, parent):
"""
If file is large add it to the large items to be processed after small task list.
else file is small add it to the small task list.
"""
if self.is_large_file(item):
self.large_items.append((item, parent))
else:
self.small_item_task_builder.visit_file(item, parent)
def is_large_file(self, item):
return item.size > self.settings.config.upload_bytes_per_chunk
def upload_large_items(self):
"""
Upload files that were too large.
"""
for local_file, parent in self.large_items:
if local_file.need_to_send:
self.process_large_file(local_file, parent)
def process_large_file(self, local_file, parent):
"""
Upload a single file using multiple processes to upload multiple chunks at the same time.
Updates local_file with it's remote_id when done.
:param local_file: LocalFile: file we are uploading
:param parent: LocalFolder/LocalProject: parent of the file
"""
file_content_sender = FileUploader(
self.settings.config, self.settings.data_service, local_file,
self.settings.watcher, self.settings.file_upload_post_processor)
remote_id = file_content_sender.upload(self.settings.project_id,
parent.kind, parent.remote_id)
local_file.set_remote_id_after_send(remote_id)