def test_handle_vectored_io_stripe(patched_gbp, patched_gfp): creds = mock.MagicMock() options = mock.MagicMock() options.mode = azmodels.StorageModes.Block store_raw_metadata = False sa = mock.MagicMock() is_file = False container = 'cont' entity = mock.MagicMock() p = '/cont/remote/path' asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') # test not first slice with mock.patch( 'blobxfer.models.metadata.vectored_io_from_metadata', side_effect=[md.VectoredStripe( next='nextpr', offset_start=0, slice_id=1, total_size=10, total_slices=10, )]): for part in asp._handle_vectored_io_stripe( creds, options, store_raw_metadata, sa, entity, is_file, container, dir=None): assert part is None # blob test with mock.patch( 'blobxfer.models.metadata.' 'vectored_io_from_metadata') as patched_vifm: patched_vifm.side_effect = [ md.VectoredStripe( next=md.VectoredNextEntry( storage_account_name='sa0', endpoint='core.windows.net', container='cont', name='path-bxslice-0', ), offset_start=0, slice_id=0, total_size=2, total_slices=2, ), md.VectoredStripe( next=None, offset_start=1, slice_id=1, total_size=2, total_slices=2, ), ] options.mode = azmodels.StorageModes.Block b0 = azure.storage.blob.models.Blob(name='path-bxslice-0') b1 = azure.storage.blob.models.Blob(name='path-bxslice-1') patched_gbp.side_effect = [b0, b1] i = 0 for part in asp._handle_vectored_io_stripe( creds, options, store_raw_metadata, sa, entity, is_file, container, dir=None): i += 1 assert i == 2 # file test with mock.patch( 'blobxfer.models.metadata.' 'vectored_io_from_metadata') as patched_vifm: patched_vifm.side_effect = [ md.VectoredStripe( next=md.VectoredNextEntry( storage_account_name='sa0', endpoint='core.windows.net', container='cont', name='path-bxslice-0', ), offset_start=0, slice_id=0, total_size=2, total_slices=2, ), md.VectoredStripe( next=None, offset_start=1, slice_id=1, total_size=2, total_slices=2, ), ] options.mode = azmodels.StorageModes.File is_file = True f0 = azure.storage.file.models.File(name='path-bxslice-0') f1 = azure.storage.file.models.File(name='path-bxslice-1') patched_gfp.side_effect = [f0, f1] i = 0 for part in asp._handle_vectored_io_stripe( creds, options, store_raw_metadata, sa, entity, is_file, container, dir=None): i += 1 assert i == 2
def test_ensure_local_destination(patched_blob, patched_file, tmpdir): downdir = tmpdir.join('down') downdir.mkdir() # no spec sources ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) # blob directory asp = azops.SourcePath() p = 'cont/remote/path' asp.add_path_with_storage_account(p, 'sa') ds.add_azure_source_path(asp) patched_blob.return_value = False ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) assert ds.destination.is_dir # blob single file + rename ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=True, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_blob.return_value = True with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) # file directory ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.File, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_file.return_value = (False, None) ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, True) assert ds.destination.is_dir # file single + rename ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.File, overwrite=True, recursive=True, rename=True, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_file.return_value = (True, mock.MagicMock()) with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False) # no read access sa = mock.MagicMock() sa.can_read_object = False creds = mock.MagicMock() creds.get_storage_account.return_value = sa ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.File, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=mock.MagicMock(), local_destination_path=models.LocalDestinationPath( str(downdir) ), ) ds.add_azure_source_path(asp) patched_file.return_value = (True, mock.MagicMock()) with pytest.raises(RuntimeError): ops.Downloader.ensure_local_destination(creds, ds, False)
def test_azuresourcepath_blobs(patched_gbp, patched_lb, patched_em): p = '/cont/remote/path' asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') options = mock.MagicMock() options.mode = azmodels.StorageModes.Auto creds = mock.MagicMock() sa = mock.MagicMock() sa.block_blob_client = mock.MagicMock() creds.get_storage_account.return_value = sa b = azure.storage.blob.models.Blob(name='name') patched_lb.side_effect = [[b]] patched_em.encryption_metadata_exists = mock.MagicMock() patched_em.encryption_metadata_exists.return_value = False # test no read access sa.can_read_object = False with pytest.raises(RuntimeError): next(asp.files(creds, options, False)) sa.can_read_object = True # test normal path i = 0 for file in asp.files(creds, options, False): i += 1 assert file.name == 'name' assert file.encryption_metadata is None assert i == 1 # test no container list perm sa.can_list_container_objects = False patched_gbp.side_effect = [b] i = 0 for file in asp.files(creds, options, False): i += 1 assert file.name == 'name' assert file.encryption_metadata is None assert i == 1 # test no container list perm, nonexistent patched_gbp.side_effect = [None] i = 0 for file in asp.files(creds, options, False): i += 1 assert i == 0 # test no container list perm, filter dry run asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') asp.add_includes(['zzz']) patched_gbp.side_effect = [b] assert len(list(asp.files(creds, options, True))) == 0 # test no container list perm, no vio return with mock.patch('blobxfer.operations.azure.SourcePath.' '_handle_vectored_io_stripe') as patched_hvios: patched_hvios.side_effect = [[None]] asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') patched_gbp.side_effect = [b] assert len(list(asp.files(creds, options, False))) == 0 sa.can_list_container_objects = True # test filter asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') asp.add_includes(['zzz']) patched_lb.side_effect = [[b]] assert len(list(asp.files(creds, options, True))) == 0 # test no vio return with mock.patch('blobxfer.operations.azure.SourcePath.' '_handle_vectored_io_stripe') as patched_hvios: patched_hvios.side_effect = [[None]] asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') patched_lb.side_effect = [[b]] assert len(list(asp.files(creds, options, False))) == 0 be = azure.storage.blob.models.Blob(name='name') be.metadata = {'encryptiondata': {'a': 'b'}} patched_lb.side_effect = [[be]] patched_em.encryption_metadata_exists.return_value = True patched_em.convert_from_json = mock.MagicMock() i = 0 for file in asp.files(creds, options, False): i += 1 assert file.name == 'name' assert file.encryption_metadata is not None assert i == 1
def test_azuresourcepath_files(patched_cisf, patched_gfp, patched_lf, patched_em): p = 'cont/name' asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') options = mock.MagicMock() options.mode = azmodels.StorageModes.File creds = mock.MagicMock() sa = mock.MagicMock() sa.file_client = mock.MagicMock() creds.get_storage_account.return_value = sa f = azure.storage.file.models.File(name='name') patched_cisf.return_value = (False, None) patched_lf.side_effect = [[f]] patched_em.encryption_metadata_exists = mock.MagicMock() patched_em.encryption_metadata_exists.return_value = False # test no read access sa.can_read_object = False with pytest.raises(RuntimeError): next(asp.files(creds, options, False)) sa.can_read_object = True # test normal container path i = 0 for file in asp.files(creds, options, False): i += 1 assert pathlib.Path(file.name) == pathlib.Path('name') assert file.encryption_metadata is None assert i == 1 p = '/cont/remote/path' asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') options = mock.MagicMock() options.mode = azmodels.StorageModes.File creds = mock.MagicMock() sa = mock.MagicMock() sa.file_client = mock.MagicMock() creds.get_storage_account.return_value = sa f = azure.storage.file.models.File(name='remote/name') patched_cisf.return_value = (False, None) patched_lf.side_effect = [[f]] patched_em.encryption_metadata_exists = mock.MagicMock() patched_em.encryption_metadata_exists.return_value = False # test normal subdir path i = 0 for file in asp.files(creds, options, False): i += 1 assert pathlib.Path(file.name) == pathlib.Path('remote/name') assert file.encryption_metadata is None assert i == 1 # test no container list perm sa.can_list_container_objects = False patched_gfp.side_effect = [f] i = 0 for file in asp.files(creds, options, False): i += 1 assert pathlib.Path(file.name) == pathlib.Path('remote/name') assert file.encryption_metadata is None assert i == 1 # test no container list perm, nonexistent patched_gfp.side_effect = [None] i = 0 for file in asp.files(creds, options, False): i += 1 assert i == 0 # test no container list perm, filter dry run asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') asp.add_includes(['zzz']) patched_cisf.return_value = (True, f) patched_gfp.side_effect = [f] assert len(list(asp.files(creds, options, True))) == 0 # test no container list perm, no vio return with mock.patch('blobxfer.operations.azure.SourcePath.' '_handle_vectored_io_stripe') as patched_hvios: patched_hvios.side_effect = [[None]] asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') patched_gfp.side_effect = [f] assert len(list(asp.files(creds, options, False))) == 0 sa.can_list_container_objects = True # test filter asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') asp.add_includes(['zzz']) patched_cisf.return_value = (True, f) patched_lf.side_effect = [[f]] assert len(list(asp.files(creds, options, True))) == 0 # test no vio return with mock.patch('blobxfer.operations.azure.SourcePath.' '_handle_vectored_io_stripe') as patched_hvios: patched_hvios.side_effect = [[None]] asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') patched_lf.side_effect = [[f]] assert len(list(asp.files(creds, options, False))) == 0 # test encrypted asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') fe = azure.storage.file.models.File(name='remote/name') fe.metadata = {'encryptiondata': {'a': 'b'}} patched_lf.side_effect = [[fe]] patched_em.encryption_metadata_exists.return_value = True patched_em.convert_from_json = mock.MagicMock() i = 0 for file in asp.files(creds, options, True): i += 1 assert pathlib.Path(file.name) == pathlib.Path('remote/name') assert file.encryption_metadata is not None assert i == 1
def test_downloadspecification(): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) asp = azops.SourcePath() p = 'some/remote/path' asp.add_path_with_storage_account(p, 'sa') ds.add_azure_source_path(asp) assert ds.options.check_file_md5 assert not ds.skip_on.lmt_ge assert ds.destination.path == pathlib.Path('dest') assert len(ds.sources) == 1 assert p in ds.sources[0]._path_map assert ds.sources[0]._path_map[p] == 'sa' with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=False, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=0, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=False, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) if util.on_windows(): patch_func = 'time.sleep' else: patch_func = 'os.getuid' with mock.patch(patch_func) as patched_getuid: patched_getuid.return_value = 1 with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=-1, delete_extraneous_destination=False, delete_only=False, max_single_object_concurrency=8, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_properties=options.FileProperties( attributes=True, cache_control=None, content_type=None, lmt=False, md5=None, ), rsa_private_key=None, strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), )
def test_downloadspecification(): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_attributes=False, rsa_private_key=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) asp = azops.SourcePath() p = 'some/remote/path' asp.add_path_with_storage_account(p, 'sa') ds.add_azure_source_path(asp) assert ds.options.check_file_md5 assert not ds.skip_on.lmt_ge assert ds.destination.path == pathlib.Path('dest') assert len(ds.sources) == 1 assert p in ds.sources[0]._path_map assert ds.sources[0]._path_map[p] == 'sa' with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=False, chunk_size_bytes=4194304, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_attributes=False, rsa_private_key=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), ) with pytest.raises(ValueError): ds = models.Specification( download_options=options.Download( check_file_md5=True, chunk_size_bytes=-1, delete_extraneous_destination=False, mode=azmodels.StorageModes.Auto, overwrite=True, recursive=True, rename=False, restore_file_attributes=True, rsa_private_key=None, ), skip_on_options=options.SkipOn( filesize_match=True, lmt_ge=False, md5_match=True, ), local_destination_path=models.LocalDestinationPath('dest'), )