def test_stuff(self, mock_progress_bar):
progress_printer = ProgressPrinter(total=10, msg_verb='sending')
# pretend we just created a project
mock_project = Mock(kind=KindType.project_str, path='')
progress_printer.transferring_item(item=mock_project, increment_amt=1)
mock_progress_bar.return_value.update.assert_called_with(
10, 'sending project')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we just created a folder
mock_project = Mock(kind=KindType.folder_str, path='/data')
progress_printer.transferring_item(item=mock_project, increment_amt=2)
mock_progress_bar.return_value.update.assert_called_with(
30, 'sending data')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we get stuck waiting for project uploads to be ready
progress_printer.start_waiting()
mock_progress_bar.return_value.show_waiting.assert_called_with(
'Waiting for project to become ready for sending')
mock_progress_bar.reset_mock()
# pretend project uploads are ready
progress_printer.done_waiting()
mock_progress_bar.return_value.show_running.assert_called()
mock_progress_bar.reset_mock()
# pretend we uploaded a file
mock_project = Mock(kind=KindType.file_str, path='/data/log.txt')
progress_printer.transferring_item(item=mock_project, increment_amt=2)
mock_progress_bar.return_value.update.assert_called_with(
50, 'sending log.txt')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we are finished uploading
progress_printer.finished()
mock_progress_bar.return_value.set_state.assert_called_with(
mock_progress_bar.STATE_DONE)
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
def test_stuff(self, mock_progress_bar):
progress_printer = ProgressPrinter(total=10, msg_verb='sending')
# pretend we just created a project
mock_project = Mock(kind=KindType.project_str, path='')
progress_printer.transferring_item(item=mock_project, increment_amt=1)
mock_progress_bar.return_value.update.assert_called_with(10, 'sending project')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we just created a folder
mock_project = Mock(kind=KindType.folder_str, path='/data')
progress_printer.transferring_item(item=mock_project, increment_amt=2)
mock_progress_bar.return_value.update.assert_called_with(30, 'sending data')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we get stuck waiting for project uploads to be ready
progress_printer.start_waiting()
mock_progress_bar.return_value.show_waiting.assert_called_with('Waiting for project to become ready for sending')
mock_progress_bar.reset_mock()
# pretend project uploads are ready
progress_printer.done_waiting()
mock_progress_bar.return_value.show_running.assert_called()
mock_progress_bar.reset_mock()
# pretend we uploaded a file
mock_project = Mock(kind=KindType.file_str, path='/data/log.txt')
progress_printer.transferring_item(item=mock_project, increment_amt=2)
mock_progress_bar.return_value.update.assert_called_with(50, 'sending log.txt')
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
# pretend we are finished uploading
progress_printer.finished()
mock_progress_bar.return_value.set_state.assert_called_with(mock_progress_bar.STATE_DONE)
mock_progress_bar.return_value.show.assert_called()
mock_progress_bar.reset_mock()
class ProjectDownload(object):
"""
Creates local version of remote content.
"""
def __init__(self, remote_store, project, dest_directory, path_filter, file_download_pre_processor=None):
"""
Setup for downloading a remote project.
:param remote_store: RemoteStore: which remote store to download the project from
:param project: RemoteProject: project to download
:param dest_directory: str: path to where we will save the project contents
:param path_filter: PathFilter: determines which files will be downloaded
:param file_download_pre_processor: object: has run(data_service, RemoteFile) method to run before downloading
"""
self.remote_store = remote_store
self.project = project
self.dest_directory = dest_directory
self.path_filter = path_filter
self.file_download_pre_processor = file_download_pre_processor
self.watcher = None
def run(self):
"""
Download the contents of the specified project name or id to dest_directory.
"""
self.walk_project(self.project)
def walk_project(self, project):
"""
For each project, folder, and files send to remote store if necessary.
:param project: LocalProject project who's contents we want to walk/send.
"""
counter = RemoteContentCounter(project)
path_filtered_project = PathFilteredProject(self.path_filter, counter)
path_filtered_project.run(project) # calls visit_project, visit_folder, visit_file in RemoteContentCounter
self.watcher = ProgressPrinter(counter.count, msg_verb='downloading')
path_filtered_project = PathFilteredProject(self.path_filter, self)
path_filtered_project.run(project) # calls visit_project, visit_folder, visit_file below
self.watcher.finished()
warnings = self.check_warnings()
if warnings:
self.watcher.show_warning(warnings)
def try_create_dir(self, remote_path):
"""
Try to create a directory if it doesn't exist and raise error if there is a non-directory with the same name.
:param path: str path to the directory
:param remote_path: str path as it exists on the remote server
"""
path = os.path.join(self.dest_directory, remote_path)
if not os.path.exists(path):
os.mkdir(path)
elif not os.path.isdir(path):
ValueError("Unable to create directory:" + path + " because a file already exists with the same name.")
def visit_project(self, item):
"""
Create the parent directory if necessary.
:param item: RemoteProject
"""
self.try_create_dir(item.remote_path)
def visit_folder(self, item, parent):
"""
Make directory for item.
:param item: RemoteFolder item we want create a directory for.
:param parent: RemoteProject/RemoteFolder parent of item
"""
self.try_create_dir(item.remote_path)
def visit_file(self, item, parent):
"""
Download the file associated with item and make sure we received all of it.
:param item: RemoteFile file we will download
:param parent: RemoteProject/RemoteFolder parent of item
"""
if self.file_download_pre_processor:
self.file_download_pre_processor.run(self.remote_store.data_service, item)
path = os.path.join(self.dest_directory, item.remote_path)
if self.file_exists_with_same_hash(item, path):
# Update progress bar skipping this file
self.watcher.transferring_item(item, increment_amt=item.size)
else:
downloader = FileDownloader(self.remote_store.config, item, path, self.watcher)
downloader.run()
ProjectDownload.check_file_size(item, path)
@staticmethod
def file_exists_with_same_hash(item, path):
if os.path.exists(path):
hash_data = PathData(path).get_hash()
return hash_data.matches(item.hash_alg, item.file_hash)
return False
@staticmethod
def check_file_size(item, path):
"""
Raise an error if we didn't get all of the file.
:param item: RemoteFile file we tried to download
:param path: str path where we downloaded the file to
"""
stat_info = os.stat(path)
if stat_info.st_size != item.size:
format_str = "Error occurred downloading {}. Got a file size {}. Expected file size:{}"
msg = format_str.format(path, stat_info.st_size, item.size)
raise ValueError(msg)
def check_warnings(self):
unused_paths = self.path_filter.get_unused_paths()
if unused_paths:
return 'WARNING: Path(s) not found: {}.'.format(','.join(unused_paths))
return None