def test_handle_vectored_io_stripe(patched_gbp, patched_gfp):
creds = mock.MagicMock()
options = mock.MagicMock()
options.mode = azmodels.StorageModes.Block
store_raw_metadata = False
sa = mock.MagicMock()
is_file = False
container = 'cont'
entity = mock.MagicMock()
p = '/cont/remote/path'
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
# test not first slice
with mock.patch(
'blobxfer.models.metadata.vectored_io_from_metadata',
side_effect=[md.VectoredStripe(
next='nextpr',
offset_start=0,
slice_id=1,
total_size=10,
total_slices=10,
)]):
for part in asp._handle_vectored_io_stripe(
creds, options, store_raw_metadata, sa, entity, is_file,
container, dir=None):
assert part is None
# blob test
with mock.patch(
'blobxfer.models.metadata.'
'vectored_io_from_metadata') as patched_vifm:
patched_vifm.side_effect = [
md.VectoredStripe(
next=md.VectoredNextEntry(
storage_account_name='sa0',
endpoint='core.windows.net',
container='cont',
name='path-bxslice-0',
),
offset_start=0,
slice_id=0,
total_size=2,
total_slices=2,
),
md.VectoredStripe(
next=None,
offset_start=1,
slice_id=1,
total_size=2,
total_slices=2,
),
]
options.mode = azmodels.StorageModes.Block
b0 = azure.storage.blob.models.Blob(name='path-bxslice-0')
b1 = azure.storage.blob.models.Blob(name='path-bxslice-1')
patched_gbp.side_effect = [b0, b1]
i = 0
for part in asp._handle_vectored_io_stripe(
creds, options, store_raw_metadata, sa, entity, is_file,
container, dir=None):
i += 1
assert i == 2
# file test
with mock.patch(
'blobxfer.models.metadata.'
'vectored_io_from_metadata') as patched_vifm:
patched_vifm.side_effect = [
md.VectoredStripe(
next=md.VectoredNextEntry(
storage_account_name='sa0',
endpoint='core.windows.net',
container='cont',
name='path-bxslice-0',
),
offset_start=0,
slice_id=0,
total_size=2,
total_slices=2,
),
md.VectoredStripe(
next=None,
offset_start=1,
slice_id=1,
total_size=2,
total_slices=2,
),
]
options.mode = azmodels.StorageModes.File
is_file = True
f0 = azure.storage.file.models.File(name='path-bxslice-0')
f1 = azure.storage.file.models.File(name='path-bxslice-1')
patched_gfp.side_effect = [f0, f1]
i = 0
for part in asp._handle_vectored_io_stripe(
creds, options, store_raw_metadata, sa, entity, is_file,
container, dir=None):
i += 1
assert i == 2
def test_ensure_local_destination(patched_blob, patched_file, tmpdir):
downdir = tmpdir.join('down')
downdir.mkdir()
# no spec sources
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=mock.MagicMock(),
local_destination_path=models.LocalDestinationPath(
str(downdir)
),
)
with pytest.raises(RuntimeError):
ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False)
# blob directory
asp = azops.SourcePath()
p = 'cont/remote/path'
asp.add_path_with_storage_account(p, 'sa')
ds.add_azure_source_path(asp)
patched_blob.return_value = False
ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False)
assert ds.destination.is_dir
# blob single file + rename
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=True,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=mock.MagicMock(),
local_destination_path=models.LocalDestinationPath(
str(downdir)
),
)
ds.add_azure_source_path(asp)
patched_blob.return_value = True
with pytest.raises(RuntimeError):
ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False)
# file directory
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.File,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=mock.MagicMock(),
local_destination_path=models.LocalDestinationPath(
str(downdir)
),
)
ds.add_azure_source_path(asp)
patched_file.return_value = (False, None)
ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, True)
assert ds.destination.is_dir
# file single + rename
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.File,
overwrite=True,
recursive=True,
rename=True,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=mock.MagicMock(),
local_destination_path=models.LocalDestinationPath(
str(downdir)
),
)
ds.add_azure_source_path(asp)
patched_file.return_value = (True, mock.MagicMock())
with pytest.raises(RuntimeError):
ops.Downloader.ensure_local_destination(mock.MagicMock(), ds, False)
# no read access
sa = mock.MagicMock()
sa.can_read_object = False
creds = mock.MagicMock()
creds.get_storage_account.return_value = sa
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.File,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=mock.MagicMock(),
local_destination_path=models.LocalDestinationPath(
str(downdir)
),
)
ds.add_azure_source_path(asp)
patched_file.return_value = (True, mock.MagicMock())
with pytest.raises(RuntimeError):
ops.Downloader.ensure_local_destination(creds, ds, False)
def test_azuresourcepath_blobs(patched_gbp, patched_lb, patched_em):
p = '/cont/remote/path'
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
options = mock.MagicMock()
options.mode = azmodels.StorageModes.Auto
creds = mock.MagicMock()
sa = mock.MagicMock()
sa.block_blob_client = mock.MagicMock()
creds.get_storage_account.return_value = sa
b = azure.storage.blob.models.Blob(name='name')
patched_lb.side_effect = [[b]]
patched_em.encryption_metadata_exists = mock.MagicMock()
patched_em.encryption_metadata_exists.return_value = False
# test no read access
sa.can_read_object = False
with pytest.raises(RuntimeError):
next(asp.files(creds, options, False))
sa.can_read_object = True
# test normal path
i = 0
for file in asp.files(creds, options, False):
i += 1
assert file.name == 'name'
assert file.encryption_metadata is None
assert i == 1
# test no container list perm
sa.can_list_container_objects = False
patched_gbp.side_effect = [b]
i = 0
for file in asp.files(creds, options, False):
i += 1
assert file.name == 'name'
assert file.encryption_metadata is None
assert i == 1
# test no container list perm, nonexistent
patched_gbp.side_effect = [None]
i = 0
for file in asp.files(creds, options, False):
i += 1
assert i == 0
# test no container list perm, filter dry run
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
asp.add_includes(['zzz'])
patched_gbp.side_effect = [b]
assert len(list(asp.files(creds, options, True))) == 0
# test no container list perm, no vio return
with mock.patch('blobxfer.operations.azure.SourcePath.'
'_handle_vectored_io_stripe') as patched_hvios:
patched_hvios.side_effect = [[None]]
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
patched_gbp.side_effect = [b]
assert len(list(asp.files(creds, options, False))) == 0
sa.can_list_container_objects = True
# test filter
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
asp.add_includes(['zzz'])
patched_lb.side_effect = [[b]]
assert len(list(asp.files(creds, options, True))) == 0
# test no vio return
with mock.patch('blobxfer.operations.azure.SourcePath.'
'_handle_vectored_io_stripe') as patched_hvios:
patched_hvios.side_effect = [[None]]
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
patched_lb.side_effect = [[b]]
assert len(list(asp.files(creds, options, False))) == 0
be = azure.storage.blob.models.Blob(name='name')
be.metadata = {'encryptiondata': {'a': 'b'}}
patched_lb.side_effect = [[be]]
patched_em.encryption_metadata_exists.return_value = True
patched_em.convert_from_json = mock.MagicMock()
i = 0
for file in asp.files(creds, options, False):
i += 1
assert file.name == 'name'
assert file.encryption_metadata is not None
assert i == 1
def test_azuresourcepath_files(patched_cisf, patched_gfp, patched_lf,
patched_em):
p = 'cont/name'
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
options = mock.MagicMock()
options.mode = azmodels.StorageModes.File
creds = mock.MagicMock()
sa = mock.MagicMock()
sa.file_client = mock.MagicMock()
creds.get_storage_account.return_value = sa
f = azure.storage.file.models.File(name='name')
patched_cisf.return_value = (False, None)
patched_lf.side_effect = [[f]]
patched_em.encryption_metadata_exists = mock.MagicMock()
patched_em.encryption_metadata_exists.return_value = False
# test no read access
sa.can_read_object = False
with pytest.raises(RuntimeError):
next(asp.files(creds, options, False))
sa.can_read_object = True
# test normal container path
i = 0
for file in asp.files(creds, options, False):
i += 1
assert pathlib.Path(file.name) == pathlib.Path('name')
assert file.encryption_metadata is None
assert i == 1
p = '/cont/remote/path'
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
options = mock.MagicMock()
options.mode = azmodels.StorageModes.File
creds = mock.MagicMock()
sa = mock.MagicMock()
sa.file_client = mock.MagicMock()
creds.get_storage_account.return_value = sa
f = azure.storage.file.models.File(name='remote/name')
patched_cisf.return_value = (False, None)
patched_lf.side_effect = [[f]]
patched_em.encryption_metadata_exists = mock.MagicMock()
patched_em.encryption_metadata_exists.return_value = False
# test normal subdir path
i = 0
for file in asp.files(creds, options, False):
i += 1
assert pathlib.Path(file.name) == pathlib.Path('remote/name')
assert file.encryption_metadata is None
assert i == 1
# test no container list perm
sa.can_list_container_objects = False
patched_gfp.side_effect = [f]
i = 0
for file in asp.files(creds, options, False):
i += 1
assert pathlib.Path(file.name) == pathlib.Path('remote/name')
assert file.encryption_metadata is None
assert i == 1
# test no container list perm, nonexistent
patched_gfp.side_effect = [None]
i = 0
for file in asp.files(creds, options, False):
i += 1
assert i == 0
# test no container list perm, filter dry run
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
asp.add_includes(['zzz'])
patched_cisf.return_value = (True, f)
patched_gfp.side_effect = [f]
assert len(list(asp.files(creds, options, True))) == 0
# test no container list perm, no vio return
with mock.patch('blobxfer.operations.azure.SourcePath.'
'_handle_vectored_io_stripe') as patched_hvios:
patched_hvios.side_effect = [[None]]
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
patched_gfp.side_effect = [f]
assert len(list(asp.files(creds, options, False))) == 0
sa.can_list_container_objects = True
# test filter
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
asp.add_includes(['zzz'])
patched_cisf.return_value = (True, f)
patched_lf.side_effect = [[f]]
assert len(list(asp.files(creds, options, True))) == 0
# test no vio return
with mock.patch('blobxfer.operations.azure.SourcePath.'
'_handle_vectored_io_stripe') as patched_hvios:
patched_hvios.side_effect = [[None]]
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
patched_lf.side_effect = [[f]]
assert len(list(asp.files(creds, options, False))) == 0
# test encrypted
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
fe = azure.storage.file.models.File(name='remote/name')
fe.metadata = {'encryptiondata': {'a': 'b'}}
patched_lf.side_effect = [[fe]]
patched_em.encryption_metadata_exists.return_value = True
patched_em.convert_from_json = mock.MagicMock()
i = 0
for file in asp.files(creds, options, True):
i += 1
assert pathlib.Path(file.name) == pathlib.Path('remote/name')
assert file.encryption_metadata is not None
assert i == 1
def test_downloadspecification():
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
content_type=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
asp = azops.SourcePath()
p = 'some/remote/path'
asp.add_path_with_storage_account(p, 'sa')
ds.add_azure_source_path(asp)
assert ds.options.check_file_md5
assert not ds.skip_on.lmt_ge
assert ds.destination.path == pathlib.Path('dest')
assert len(ds.sources) == 1
assert p in ds.sources[0]._path_map
assert ds.sources[0]._path_map[p] == 'sa'
with pytest.raises(ValueError):
ds = models.Specification(
download_options=options.Download(
check_file_md5=False,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
content_type=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
with pytest.raises(ValueError):
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
max_single_object_concurrency=0,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
content_type=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
if util.on_windows():
patch_func = 'time.sleep'
else:
patch_func = 'os.getuid'
with mock.patch(patch_func) as patched_getuid:
patched_getuid.return_value = 1
with pytest.raises(ValueError):
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=-1,
delete_extraneous_destination=False,
delete_only=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=True,
cache_control=None,
content_type=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
def test_downloadspecification():
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_attributes=False,
rsa_private_key=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
asp = azops.SourcePath()
p = 'some/remote/path'
asp.add_path_with_storage_account(p, 'sa')
ds.add_azure_source_path(asp)
assert ds.options.check_file_md5
assert not ds.skip_on.lmt_ge
assert ds.destination.path == pathlib.Path('dest')
assert len(ds.sources) == 1
assert p in ds.sources[0]._path_map
assert ds.sources[0]._path_map[p] == 'sa'
with pytest.raises(ValueError):
ds = models.Specification(
download_options=options.Download(
check_file_md5=False,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_attributes=False,
rsa_private_key=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
with pytest.raises(ValueError):
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=-1,
delete_extraneous_destination=False,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_attributes=True,
rsa_private_key=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
