def create_upload_options(storage_mode=azmodels.StorageModes.Block): return blobxfer.api.UploadOptions( access_tier=None, chunk_size_bytes=0, delete_extraneous_destination=False, mode=storage_mode, one_shot_bytes=0, overwrite=True, recursive=True, rename=False, rsa_public_key=None, stdin_as_page_blob_size=None, store_file_properties=options.FileProperties( attributes=False, md5=False, ), strip_components=0, vectored_io=blobxfer.api.VectoredIoOptions(stripe_chunk_size_bytes=0, distribution_mode='stripe'))
def create_download_options(storage_mode=azmodels.StorageModes.Block): return blobxfer.api.DownloadOptions( delete_only=False, max_single_object_concurrency=8, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, content_type=None, lmt=False, md5=None, ), check_file_md5=False, chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=storage_mode, overwrite=True, recursive=True, rename=False, # restore_file_attributes=False, rsa_private_key=None, strip_components=0)
def test_ensure_local_destination(patched_blob, patched_file, tmpdir): downdir = tmpdir.join('down') downdir.mkdir() # no spec sources ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) # blob directory asp = azops.SourcePath() p = 'cont/remote/path' asp.add_path_with_storage_account(p, 'sa') ds.add_azure_source_path(asp) patched_blob.return_value = False ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) assert ds.destination.is_dir # blob single file + rename ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=True, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_blob.return_value = True with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) # file directory ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.File, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_file.return_value = (False, None) ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, True) assert ds.destination.is_dir # file single + rename ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.File, overwrite=True, recursive=True, rename=True, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_file.return_value = (True, mock.MagicMock()) with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) # no read access sa = mock.MagicMock() sa.can_read_object = False creds = mock.MagicMock() creds.get_storage_account.return_value = sa ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.File, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_file.return_value = (True, mock.MagicMock()) with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(creds, ds, False)
def test_check_download_conditions(tmpdir): ap = tmpdir.join('a') ap.write('abc') ep = pathlib.Path(str(ap)) nep = pathlib.Path(str(tmpdir.join('nep'))) ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=False, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=True, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds) result = d._check_download_conditions(nep, mock.MagicMock()) assert result == ops.DownloadAction.Download rfile = mock.MagicMock() rfile.vectored_io = None result = d._check_download_conditions(nep, rfile) assert result == ops.DownloadAction.Download result = d._check_download_conditions(ep, mock.MagicMock()) assert result == ops.DownloadAction.Skip ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=True, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds) rfile = mock.MagicMock() rfile.md5 = 'abc' rfile._encryption = None result = d._check_download_conditions(ep, rfile) assert result == ops.DownloadAction.CheckMd5 ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=False, lmt_ge=False, md5_match=False, ), local_destination_path=models.LocalDestinationPath('dest'), ) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds) result = d._check_download_conditions(ep, mock.MagicMock()) assert result == ops.DownloadAction.Download ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=False, ), local_destination_path=models.LocalDestinationPath('dest'), ) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds) rfile = azmodels.StorageEntity('cont') rfile._size = util.page_align_content_length(ep.stat().st_size) rfile._mode = azmodels.StorageModes.Page rfile._encryption = None result = d._check_download_conditions(ep, rfile) assert result == ops.DownloadAction.Skip rfile._size = ep.stat().st_size rfile._mode = azmodels.StorageModes.Page result = d._check_download_conditions(ep, rfile) assert result == ops.DownloadAction.Download ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=False, lmt_ge=True, md5_match=False, ), local_destination_path=models.LocalDestinationPath('dest'), ) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds) rfile = azmodels.StorageEntity('cont') rfile._lmt = datetime.datetime.now(dateutil.tz.tzutc()) + \ datetime.timedelta(days=1) rfile._encryption = None result = d._check_download_conditions(ep, rfile) assert result == ops.DownloadAction.Download rfile._lmt = datetime.datetime.now(dateutil.tz.tzutc()) - \ datetime.timedelta(days=1) result = d._check_download_conditions(ep, rfile) assert result == ops.DownloadAction.Skip
def test_output_parameters(): go = mock.MagicMock() go.log_file = 'abc' spec = modelsdl.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_attributes=False, rsa_private_key=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=mock.MagicMock(), ) ops.output_parameters(go, spec) assert util.is_not_empty(go.log_file) spec = modelsul.Specification(upload_options=options.Upload( chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, one_shot_bytes=0, overwrite=True, recursive=True, rename=False, rsa_public_key=None, stdin_as_page_blob_size=0, store_file_properties=options.FileProperties( attributes=True, md5=True, ), strip_components=0, vectored_io=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_source_path=mock.MagicMock()) ops.output_parameters(go, spec) assert util.is_not_empty(go.log_file) spec = modelssc.Specification(synccopy_options=options.SyncCopy( delete_extraneous_destination=False, dest_mode=azmodels.StorageModes.Auto, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, )) ops.output_parameters(go, spec) assert util.is_not_empty(go.log_file)
def test_downloadspecification(): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) asp = azops.SourcePath() p = 'some/remote/path' asp.add_path_with_storage_account(p, 'sa') ds.add_azure_source_path(asp) assert ds.options.check_file_md5 assert not ds.skip_on.lmt_ge assert ds.destination.path == pathlib.Path('dest') assert len(ds.sources) == 1 assert p in ds.sources[0]._path_map assert ds.sources[0]._path_map[p] == 'sa' with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=False, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=0, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) if util.on_windows(): patch_func = 'time.sleep' else: patch_func = 'os.getuid' with mock.patch(patch_func) as patched_getuid: patched_getuid.return_value = 1 with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=-1, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=True, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), )
def test_specification(tmpdir): lsp = upload.LocalSourcePath() lsp.add_paths(['-', '/dev/stdin']) with pytest.raises(ValueError): upload.Specification( upload_options=options.Upload( access_tier=None, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, mode=azmodels.StorageModes.Auto, one_shot_bytes=0, overwrite=True, recursive=True, rename=True, rsa_public_key=None, stdin_as_page_blob_size=0, store_file_properties=options.FileProperties( attributes=True, cache_control='cc', content_type='ct', lmt=None, md5=True, ), strip_components=0, vectored_io=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_source_path=lsp, ) lsp = upload.LocalSourcePath() lsp.add_path(str(tmpdir)) with pytest.raises(ValueError): upload.Specification( upload_options=options.Upload( access_tier=None, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, mode=azmodels.StorageModes.Auto, one_shot_bytes=0, overwrite=True, recursive=True, rename=True, rsa_public_key=None, stdin_as_page_blob_size=0, store_file_properties=options.FileProperties( attributes=True, cache_control='cc', content_type='ct', lmt=None, md5=True, ), strip_components=0, vectored_io=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_source_path=lsp, ) lsp = upload.LocalSourcePath() lsp.add_path(str(tmpdir)) with pytest.raises(ValueError): upload.Specification( upload_options=options.Upload( access_tier=None, chunk_size_bytes=-1, delete_extraneous_destination=False, delete_only=False, mode=azmodels.StorageModes.Auto, one_shot_bytes=0, overwrite=True, recursive=True, rename=False, rsa_public_key=None, stdin_as_page_blob_size=0, store_file_properties=options.FileProperties( attributes=True, cache_control='cc', content_type='ct', lmt=None, md5=True, ), strip_components=0, vectored_io=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_source_path=lsp, ) with pytest.raises(ValueError): upload.Specification( upload_options=options.Upload( access_tier=None, chunk_size_bytes=upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES + 1, delete_extraneous_destination=False, delete_only=False, mode=azmodels.StorageModes.Auto, one_shot_bytes=0, overwrite=True, recursive=True, rename=False, rsa_public_key=None, stdin_as_page_blob_size=0, store_file_properties=options.FileProperties( attributes=True, cache_control='cc', content_type='ct', lmt=None, md5=True, ), strip_components=0, vectored_io=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_source_path=lsp, ) with pytest.raises(ValueError): upload.Specification( upload_options=options.Upload( access_tier=None, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, mode=azmodels.StorageModes.Auto, one_shot_bytes=-1, overwrite=True, recursive=True, rename=False, rsa_public_key=None, stdin_as_page_blob_size=0, store_file_properties=options.FileProperties( attributes=True, cache_control='cc', content_type='ct', lmt=None, md5=True, ), strip_components=0, vectored_io=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_source_path=lsp, ) with pytest.raises(ValueError): upload.Specification( upload_options=options.Upload( access_tier=None, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, mode=azmodels.StorageModes.Auto, one_shot_bytes=upload._MAX_BLOCK_BLOB_ONESHOT_BYTES + 1, overwrite=True, recursive=True, rename=False, rsa_public_key=None, stdin_as_page_blob_size=0, store_file_properties=options.FileProperties( attributes=True, cache_control=None, content_type=None, lmt=None, md5=True, ), strip_components=0, vectored_io=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_source_path=lsp, ) spec = upload.Specification( upload_options=options.Upload( access_tier=None, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, mode=azmodels.StorageModes.Auto, one_shot_bytes=0, overwrite=True, recursive=True, rename=False, rsa_public_key=None, stdin_as_page_blob_size=0, store_file_properties=options.FileProperties( attributes=True, cache_control=None, content_type=None, lmt=None, md5=True, ), strip_components=0, vectored_io=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_source_path=lsp, ) spec.add_azure_destination_path(azops.DestinationPath()) assert len(spec.destinations) == 1
def test_downloadspecification(): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) asp = azops.SourcePath() p = 'some/remote/path' asp.add_path_with_storage_account(p, 'sa') ds.add_azure_source_path(asp) assert ds.options.check_file_md5 assert not ds.skip_on.lmt_ge assert ds.destination.path == pathlib.Path('dest') assert len(ds.sources) == 1 assert p in ds.sources[0]._path_map assert ds.sources[0]._path_map[p] == 'sa' with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=False, chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=-1, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=True, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), )