def run(self):
"""
Upload different items within local_project to remote store showing a progress bar.
"""
progress_printer = ProgressPrinter(self.different_items.total_items(),
msg_verb='sending')
upload_settings = UploadSettings(self.config,
self.remote_store.data_service,
progress_printer, self.project_name,
self.file_upload_post_processor)
project_uploader = ProjectUploader(upload_settings)
project_uploader.run(self.local_project)
progress_printer.finished()
def run(self):
"""
Download the contents of the specified project name or id to dest_directory.
"""
files_to_download = self.get_files_to_download()
total_files_size = self.get_total_files_size(files_to_download)
if self.file_download_pre_processor:
self.run_preprocessor(files_to_download)
self.try_create_dir(self.dest_directory)
watcher = ProgressPrinter(total_files_size, msg_verb='downloading')
self.download_files(files_to_download, watcher)
watcher.finished()
warnings = self.check_warnings()
if warnings:
watcher.show_warning(warnings)
def test_stuff(self, mock_progress_bar):
progress_printer = ProgressPrinter(total=10, msg_verb='sending')
# pretend we just created a project
mock_project = Mock(kind=KindType.project_str, path='')
progress_printer.transferring_item(item=mock_project, increment_amt=1)
mock_progress_bar.return_value.update.assert_called_with(
10, 'sending project')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we just created a folder
mock_project = Mock(kind=KindType.folder_str, path='/data')
progress_printer.transferring_item(item=mock_project, increment_amt=2)
mock_progress_bar.return_value.update.assert_called_with(
30, 'sending data')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we get stuck waiting for project uploads to be ready
progress_printer.start_waiting()
mock_progress_bar.return_value.show_waiting.assert_called_with(
'Waiting for project to become ready for sending')
mock_progress_bar.reset_mock()
# pretend project uploads are ready
progress_printer.done_waiting()
mock_progress_bar.return_value.show_running.assert_called()
mock_progress_bar.reset_mock()
# pretend we uploaded a file
mock_project = Mock(kind=KindType.file_str, path='/data/log.txt')
progress_printer.transferring_item(item=mock_project, increment_amt=2)
mock_progress_bar.return_value.update.assert_called_with(
50, 'sending log.txt')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we are finished uploading
progress_printer.finished()
mock_progress_bar.return_value.set_state.assert_called_with(
mock_progress_bar.STATE_DONE)
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
def test_stuff(self, mock_progress_bar):
progress_printer = ProgressPrinter(total=10, msg_verb='sending')
# pretend we just created a project
mock_project = Mock(kind=KindType.project_str, path='')
progress_printer.transferring_item(item=mock_project, increment_amt=1)
mock_progress_bar.return_value.update.assert_called_with(10, 'sending project')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we just created a folder
mock_project = Mock(kind=KindType.folder_str, path='/data')
progress_printer.transferring_item(item=mock_project, increment_amt=2)
mock_progress_bar.return_value.update.assert_called_with(30, 'sending data')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we get stuck waiting for project uploads to be ready
progress_printer.start_waiting()
mock_progress_bar.return_value.show_waiting.assert_called_with('Waiting for project to become ready for sending')
mock_progress_bar.reset_mock()
# pretend project uploads are ready
progress_printer.done_waiting()
mock_progress_bar.return_value.show_running.assert_called()
mock_progress_bar.reset_mock()
# pretend we uploaded a file
mock_project = Mock(kind=KindType.file_str, path='/data/log.txt')
progress_printer.transferring_item(item=mock_project, increment_amt=2)
mock_progress_bar.return_value.update.assert_called_with(50, 'sending log.txt')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we are finished uploading
progress_printer.finished()
mock_progress_bar.return_value.set_state.assert_called_with(mock_progress_bar.STATE_DONE)
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
class ProjectDownload(object):
"""
Creates local version of remote content.
"""
def __init__(self, remote_store, project_name, dest_directory,
path_filter):
"""
Setup for downloading a remote project.
:param remote_store: RemoteStore: which remote store to download the project from
:param project_name: str: name of the project to download
:param dest_directory: str: path to where we will save the project contents
:param path_filter: PathFilter: determines which files will be downloaded
"""
self.remote_store = remote_store
self.project_name = project_name
self.dest_directory = dest_directory
self.path_filter = path_filter
self.watcher = None
def run(self):
"""
Download the contents of the specified project_name to dest_directory.
"""
remote_project = self.remote_store.fetch_remote_project(
self.project_name, must_exist=True)
self.walk_project(remote_project)
def walk_project(self, project):
"""
For each project, folder, and files send to remote store if necessary.
:param project: LocalProject project who's contents we want to walk/send.
"""
counter = RemoteContentCounter(project)
path_filtered_project = PathFilteredProject(self.path_filter, counter)
path_filtered_project.run(
project
) # calls visit_project, visit_folder, visit_file in RemoteContentCounter
self.watcher = ProgressPrinter(counter.count, msg_verb='downloading')
path_filtered_project = PathFilteredProject(self.path_filter, self)
path_filtered_project.run(
project) # calls visit_project, visit_folder, visit_file below
self.watcher.finished()
warnings = self.check_warnings()
if warnings:
self.watcher.show_warning(warnings)
def try_create_dir(self, remote_path):
"""
Try to create a directory if it doesn't exist and raise error if there is a non-directory with the same name.
:param path: str path to the directory
:param remote_path: str path as it exists on the remote server
"""
path = os.path.join(self.dest_directory, remote_path)
if not os.path.exists(path):
os.mkdir(path)
elif not os.path.isdir(path):
ValueError("Unable to create directory:" + path +
" because a file already exists with the same name.")
def visit_project(self, item):
"""
Create the parent directory if necessary.
:param item: RemoteProject
"""
self.try_create_dir(item.remote_path)
def visit_folder(self, item, parent):
"""
Make directory for item.
:param item: RemoteFolder item we want create a directory for.
:param parent: RemoteProject/RemoteFolder parent of item
"""
self.try_create_dir(item.remote_path)
def visit_file(self, item, parent):
"""
Download the file associated with item and make sure we received all of it.
:param item: RemoteFile file we will download
:param parent: RemoteProject/RemoteFolder parent of item
"""
path = os.path.join(self.dest_directory, item.remote_path)
url_json = self.remote_store.data_service.get_file_url(item.id).json()
downloader = FileDownloader(self.remote_store.config, item, url_json,
path, self.watcher)
downloader.run()
ProjectDownload.check_file_size(item, path)
@staticmethod
def check_file_size(item, path):
"""
Raise an error if we didn't get all of the file.
:param item: RemoteFile file we tried to download
:param path: str path where we downloaded the file to
"""
stat_info = os.stat(path)
if stat_info.st_size != item.size:
format_str = "Error occurred downloading {}. Got a file size {}. Expected file size:{}"
msg = format_str.format(path, stat_info.st_size, item.size)
raise ValueError(msg)
def check_warnings(self):
unused_paths = self.path_filter.get_unused_paths()
if unused_paths:
return 'WARNING: Path(s) not found: {}.'.format(
','.join(unused_paths))
return None