def __init__(self, settings): """ Setup to talk to the data service based on settings. :param settings: UploadSettings: settings to use for uploading. """ self.runner = TaskRunner(TaskExecutor(settings.config.upload_workers)) self.settings = settings self.small_item_task_builder = SmallItemUploadTaskBuilder( self.settings, self.runner) self.small_items = [] self.large_items = []
def test_two_adds_in_order(self): add_command = AddCommand(10, 30) add_command2 = AddCommand(4, 1) executor = TaskExecutor(10) runner = TaskRunner(executor) runner.add(None, add_command) runner.add(1, add_command2) runner.run() self.assertEqual(add_command.parent_task_result, None) self.assertEqual(add_command.result, 40) self.assertEqual(add_command2.parent_task_result, 40) self.assertEqual(add_command2.result, 5)
def test_single_add(self): add_command = AddCommand(10, 30) executor = TaskExecutor(10) runner = TaskRunner(executor) runner.add(None, add_command) runner.run() self.assertEqual(add_command.parent_task_result, None) self.assertEqual(add_command.result, 40)
def test_command_with_message(self): add_command = AddCommand(10, 30) add_command.send_message = 'ok' add_command2 = AddCommand(4, 1) add_command2.send_message = 'waiting' executor = TaskExecutor(10) runner = TaskRunner(executor) runner.add( None, add_command, ) runner.add(None, add_command2) runner.run() self.assertEqual(add_command.parent_task_result, None) self.assertEqual(add_command.result, 40) self.assertEqual(add_command2.parent_task_result, None) self.assertEqual(add_command2.result, 5) self.assertEqual(add_command.on_message_data, ['ok']) self.assertEqual(add_command2.on_message_data, ['waiting'])
class ProjectUploader(object): """ Uploads a project based on UploadSettings. """ def __init__(self, settings): """ Setup to talk to the data service based on settings. :param settings: UploadSettings: settings to use for uploading. """ self.runner = TaskRunner(TaskExecutor(settings.config.upload_workers)) self.settings = settings self.small_item_task_builder = SmallItemUploadTaskBuilder( self.settings, self.runner) self.small_items = [] self.large_items = [] def run(self, local_project): """ Upload a project by uploading project, folders, and small files then uploading the large files. :param local_project: LocalProject: project to upload """ # Walk project adding small items to runner saving large items to large_items ProjectWalker.walk_project(local_project, self) # Run small items in parallel self.runner.run() # Run parts of each large item in parallel self.upload_large_items() # Methods called by ProjectWalker.walk_project def visit_project(self, item): """ Add create project to small task list. """ self.small_item_task_builder.visit_project(item) def visit_folder(self, item, parent): """ Add create folder to small task list. """ self.small_item_task_builder.visit_folder(item, parent) def visit_file(self, item, parent): """ If file is large add it to the large items to be processed after small task list. else file is small add it to the small task list. """ if self.is_large_file(item): self.large_items.append((item, parent)) else: self.small_item_task_builder.visit_file(item, parent) def is_large_file(self, item): return item.size > self.settings.config.upload_bytes_per_chunk def upload_large_items(self): """ Upload files that were too large. """ for local_file, parent in self.large_items: if local_file.need_to_send: self.process_large_file(local_file, parent) def process_large_file(self, local_file, parent): """ Upload a single file using multiple processes to upload multiple chunks at the same time. Updates local_file with it's remote_id when done. :param local_file: LocalFile: file we are uploading :param parent: LocalFolder/LocalProject: parent of the file """ file_content_sender = FileUploader( self.settings.config, self.settings.data_service, local_file, self.settings.watcher, self.settings.file_upload_post_processor) remote_id = file_content_sender.upload(self.settings.project_id, parent.kind, parent.remote_id) local_file.set_remote_id_after_send(remote_id)