def test_ensure_local_destination(patched_blob, patched_file, tmpdir): downdir = tmpdir.join('down') downdir.mkdir() # no spec sources ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) # blob directory asp = azops.SourcePath() p = 'cont/remote/path' asp.add_path_with_storage_account(p, 'sa') ds.add_azure_source_path(asp) patched_blob.return_value = False ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) assert ds.destination.is_dir # blob single file + rename ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=True, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_blob.return_value = True with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) # file directory ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.File, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_file.return_value = (False, None) ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, True) assert ds.destination.is_dir # file single + rename ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.File, overwrite=True, recursive=True, rename=True, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_file.return_value = (True, mock.MagicMock()) with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) # no read access sa = mock.MagicMock() sa.can_read_object = False creds = mock.MagicMock() creds.get_storage_account.return_value = sa ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.File, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_file.return_value = (True, mock.MagicMock()) with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(creds, ds, False)
def test_check_download_conditions(tmpdir): ap = tmpdir.join('a') ap.write('abc') ep = pathlib.Path(str(ap)) nep = pathlib.Path(str(tmpdir.join('nep'))) ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=False, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=True, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds) result = d._check_download_conditions(nep, mock.MagicMock()) assert result == ops.DownloadAction.Download rfile = mock.MagicMock() rfile.vectored_io = None result = d._check_download_conditions(nep, rfile) assert result == ops.DownloadAction.Download result = d._check_download_conditions(ep, mock.MagicMock()) assert result == ops.DownloadAction.Skip ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=True, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds) rfile = mock.MagicMock() rfile.md5 = 'abc' rfile._encryption = None result = d._check_download_conditions(ep, rfile) assert result == ops.DownloadAction.CheckMd5 ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=False, lmt_ge=False, md5_match=False, ), local_destination_path=models.LocalDestinationPath('dest'), ) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds) result = d._check_download_conditions(ep, mock.MagicMock()) assert result == ops.DownloadAction.Download ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=False, ), local_destination_path=models.LocalDestinationPath('dest'), ) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds) rfile = azmodels.StorageEntity('cont') rfile._size = util.page_align_content_length(ep.stat().st_size) rfile._mode = azmodels.StorageModes.Page rfile._encryption = None result = d._check_download_conditions(ep, rfile) assert result == ops.DownloadAction.Skip rfile._size = ep.stat().st_size rfile._mode = azmodels.StorageModes.Page result = d._check_download_conditions(ep, rfile) assert result == ops.DownloadAction.Download ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=False, lmt_ge=True, md5_match=False, ), local_destination_path=models.LocalDestinationPath('dest'), ) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds) rfile = azmodels.StorageEntity('cont') rfile._lmt = datetime.datetime.now(dateutil.tz.tzutc()) + \ datetime.timedelta(days=1) rfile._encryption = None result = d._check_download_conditions(ep, rfile) assert result == ops.DownloadAction.Download rfile._lmt = datetime.datetime.now(dateutil.tz.tzutc()) - \ datetime.timedelta(days=1) result = d._check_download_conditions(ep, rfile) assert result == ops.DownloadAction.Skip
def test_downloadspecification(): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) asp = azops.SourcePath() p = 'some/remote/path' asp.add_path_with_storage_account(p, 'sa') ds.add_azure_source_path(asp) assert ds.options.check_file_md5 assert not ds.skip_on.lmt_ge assert ds.destination.path == pathlib.Path('dest') assert len(ds.sources) == 1 assert p in ds.sources[0]._path_map assert ds.sources[0]._path_map[p] == 'sa' with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=False, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=0, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) if util.on_windows(): patch_func = 'time.sleep' else: patch_func = 'os.getuid' with mock.patch(patch_func) as patched_getuid: patched_getuid.return_value = 1 with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=-1, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=True, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), )
def test_output_parameters(): go = mock.MagicMock() go.log_file = 'abc' spec = modelsdl.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_attributes=False, rsa_private_key=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=mock.MagicMock(), ) ops.output_parameters(go, spec) assert util.is_not_empty(go.log_file) spec = modelsul.Specification(upload_options=options.Upload( chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, one_shot_bytes=0, overwrite=True, recursive=True, rename=False, rsa_public_key=None, stdin_as_page_blob_size=0, store_file_properties=options.FileProperties( attributes=True, md5=True, ), strip_components=0, vectored_io=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_source_path=mock.MagicMock()) ops.output_parameters(go, spec) assert util.is_not_empty(go.log_file) spec = modelssc.Specification(synccopy_options=options.SyncCopy( delete_extraneous_destination=False, dest_mode=azmodels.StorageModes.Auto, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, )) ops.output_parameters(go, spec) assert util.is_not_empty(go.log_file)
def test_downloadspecification(): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_attributes=False, rsa_private_key=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) asp = azops.SourcePath() p = 'some/remote/path' asp.add_path_with_storage_account(p, 'sa') ds.add_azure_source_path(asp) assert ds.options.check_file_md5 assert not ds.skip_on.lmt_ge assert ds.destination.path == pathlib.Path('dest') assert len(ds.sources) == 1 assert p in ds.sources[0]._path_map assert ds.sources[0]._path_map[p] == 'sa' with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=False, chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_attributes=False, rsa_private_key=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=-1, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_attributes=True, rsa_private_key=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), )