def test_stuff(self, mock_progress_bar): progress_printer = ProgressPrinter(total=10, msg_verb='sending') # pretend we just created a project mock_project = Mock(kind=KindType.project_str, path='') progress_printer.transferring_item(item=mock_project, increment_amt=1) mock_progress_bar.return_value.update.assert_called_with( 10, 'sending project') mock_progress_bar.return_value.show.assert_called() mock_progress_bar.reset_mock() # pretend we just created a folder mock_project = Mock(kind=KindType.folder_str, path='/data') progress_printer.transferring_item(item=mock_project, increment_amt=2) mock_progress_bar.return_value.update.assert_called_with( 30, 'sending data') mock_progress_bar.return_value.show.assert_called() mock_progress_bar.reset_mock() # pretend we get stuck waiting for project uploads to be ready progress_printer.start_waiting() mock_progress_bar.return_value.show_waiting.assert_called_with( 'Waiting for project to become ready for sending') mock_progress_bar.reset_mock() # pretend project uploads are ready progress_printer.done_waiting() mock_progress_bar.return_value.show_running.assert_called() mock_progress_bar.reset_mock() # pretend we uploaded a file mock_project = Mock(kind=KindType.file_str, path='/data/log.txt') progress_printer.transferring_item(item=mock_project, increment_amt=2) mock_progress_bar.return_value.update.assert_called_with( 50, 'sending log.txt') mock_progress_bar.return_value.show.assert_called() mock_progress_bar.reset_mock() # pretend we are finished uploading progress_printer.finished() mock_progress_bar.return_value.set_state.assert_called_with( mock_progress_bar.STATE_DONE) mock_progress_bar.return_value.show.assert_called() mock_progress_bar.reset_mock()
def test_stuff(self, mock_progress_bar): progress_printer = ProgressPrinter(total=10, msg_verb='sending') # pretend we just created a project mock_project = Mock(kind=KindType.project_str, path='') progress_printer.transferring_item(item=mock_project, increment_amt=1) mock_progress_bar.return_value.update.assert_called_with(10, 'sending project') mock_progress_bar.return_value.show.assert_called() mock_progress_bar.reset_mock() # pretend we just created a folder mock_project = Mock(kind=KindType.folder_str, path='/data') progress_printer.transferring_item(item=mock_project, increment_amt=2) mock_progress_bar.return_value.update.assert_called_with(30, 'sending data') mock_progress_bar.return_value.show.assert_called() mock_progress_bar.reset_mock() # pretend we get stuck waiting for project uploads to be ready progress_printer.start_waiting() mock_progress_bar.return_value.show_waiting.assert_called_with('Waiting for project to become ready for sending') mock_progress_bar.reset_mock() # pretend project uploads are ready progress_printer.done_waiting() mock_progress_bar.return_value.show_running.assert_called() mock_progress_bar.reset_mock() # pretend we uploaded a file mock_project = Mock(kind=KindType.file_str, path='/data/log.txt') progress_printer.transferring_item(item=mock_project, increment_amt=2) mock_progress_bar.return_value.update.assert_called_with(50, 'sending log.txt') mock_progress_bar.return_value.show.assert_called() mock_progress_bar.reset_mock() # pretend we are finished uploading progress_printer.finished() mock_progress_bar.return_value.set_state.assert_called_with(mock_progress_bar.STATE_DONE) mock_progress_bar.return_value.show.assert_called() mock_progress_bar.reset_mock()
class ProjectDownload(object): """ Creates local version of remote content. """ def __init__(self, remote_store, project, dest_directory, path_filter, file_download_pre_processor=None): """ Setup for downloading a remote project. :param remote_store: RemoteStore: which remote store to download the project from :param project: RemoteProject: project to download :param dest_directory: str: path to where we will save the project contents :param path_filter: PathFilter: determines which files will be downloaded :param file_download_pre_processor: object: has run(data_service, RemoteFile) method to run before downloading """ self.remote_store = remote_store self.project = project self.dest_directory = dest_directory self.path_filter = path_filter self.file_download_pre_processor = file_download_pre_processor self.watcher = None def run(self): """ Download the contents of the specified project name or id to dest_directory. """ self.walk_project(self.project) def walk_project(self, project): """ For each project, folder, and files send to remote store if necessary. :param project: LocalProject project who's contents we want to walk/send. """ counter = RemoteContentCounter(project) path_filtered_project = PathFilteredProject(self.path_filter, counter) path_filtered_project.run(project) # calls visit_project, visit_folder, visit_file in RemoteContentCounter self.watcher = ProgressPrinter(counter.count, msg_verb='downloading') path_filtered_project = PathFilteredProject(self.path_filter, self) path_filtered_project.run(project) # calls visit_project, visit_folder, visit_file below self.watcher.finished() warnings = self.check_warnings() if warnings: self.watcher.show_warning(warnings) def try_create_dir(self, remote_path): """ Try to create a directory if it doesn't exist and raise error if there is a non-directory with the same name. :param path: str path to the directory :param remote_path: str path as it exists on the remote server """ path = os.path.join(self.dest_directory, remote_path) if not os.path.exists(path): os.mkdir(path) elif not os.path.isdir(path): ValueError("Unable to create directory:" + path + " because a file already exists with the same name.") def visit_project(self, item): """ Create the parent directory if necessary. :param item: RemoteProject """ self.try_create_dir(item.remote_path) def visit_folder(self, item, parent): """ Make directory for item. :param item: RemoteFolder item we want create a directory for. :param parent: RemoteProject/RemoteFolder parent of item """ self.try_create_dir(item.remote_path) def visit_file(self, item, parent): """ Download the file associated with item and make sure we received all of it. :param item: RemoteFile file we will download :param parent: RemoteProject/RemoteFolder parent of item """ if self.file_download_pre_processor: self.file_download_pre_processor.run(self.remote_store.data_service, item) path = os.path.join(self.dest_directory, item.remote_path) if self.file_exists_with_same_hash(item, path): # Update progress bar skipping this file self.watcher.transferring_item(item, increment_amt=item.size) else: downloader = FileDownloader(self.remote_store.config, item, path, self.watcher) downloader.run() ProjectDownload.check_file_size(item, path) @staticmethod def file_exists_with_same_hash(item, path): if os.path.exists(path): hash_data = PathData(path).get_hash() return hash_data.matches(item.hash_alg, item.file_hash) return False @staticmethod def check_file_size(item, path): """ Raise an error if we didn't get all of the file. :param item: RemoteFile file we tried to download :param path: str path where we downloaded the file to """ stat_info = os.stat(path) if stat_info.st_size != item.size: format_str = "Error occurred downloading {}. Got a file size {}. Expected file size:{}" msg = format_str.format(path, stat_info.st_size, item.size) raise ValueError(msg) def check_warnings(self): unused_paths = self.path_filter.get_unused_paths() if unused_paths: return 'WARNING: Path(s) not found: {}.'.format(','.join(unused_paths)) return None