def test_specification():
spec = synccopy.Specification(synccopy_options=options.SyncCopy(
access_tier=None,
delete_extraneous_destination=False,
dest_mode=azmodels.StorageModes.Auto,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
))
spec.add_azure_source_path(mock.MagicMock())
assert len(spec.sources) == 1
spec.add_azure_destination_path(mock.MagicMock())
assert len(spec.destinations) == 1
def test_check_download_conditions(tmpdir):
ap = tmpdir.join('a')
ap.write('abc')
ep = pathlib.Path(str(ap))
nep = pathlib.Path(str(tmpdir.join('nep')))
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=False,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=True,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds)
result = d._check_download_conditions(nep, mock.MagicMock())
assert result == ops.DownloadAction.Download
rfile = mock.MagicMock()
rfile.vectored_io = None
result = d._check_download_conditions(nep, rfile)
assert result == ops.DownloadAction.Download
result = d._check_download_conditions(ep, mock.MagicMock())
assert result == ops.DownloadAction.Skip
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=True,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds)
rfile = mock.MagicMock()
rfile.md5 = 'abc'
rfile._encryption = None
result = d._check_download_conditions(ep, rfile)
assert result == ops.DownloadAction.CheckMd5
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=False,
lmt_ge=False,
md5_match=False,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds)
result = d._check_download_conditions(ep, mock.MagicMock())
assert result == ops.DownloadAction.Download
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=False,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds)
rfile = azmodels.StorageEntity('cont')
rfile._size = util.page_align_content_length(ep.stat().st_size)
rfile._mode = azmodels.StorageModes.Page
rfile._encryption = None
result = d._check_download_conditions(ep, rfile)
assert result == ops.DownloadAction.Skip
rfile._size = ep.stat().st_size
rfile._mode = azmodels.StorageModes.Page
result = d._check_download_conditions(ep, rfile)
assert result == ops.DownloadAction.Download
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=False,
lmt_ge=True,
md5_match=False,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), ds)
rfile = azmodels.StorageEntity('cont')
rfile._lmt = datetime.datetime.now(dateutil.tz.tzutc()) + \
datetime.timedelta(days=1)
rfile._encryption = None
result = d._check_download_conditions(ep, rfile)
assert result == ops.DownloadAction.Download
rfile._lmt = datetime.datetime.now(dateutil.tz.tzutc()) - \
datetime.timedelta(days=1)
result = d._check_download_conditions(ep, rfile)
assert result == ops.DownloadAction.Skip
def test_output_parameters():
go = mock.MagicMock()
go.log_file = 'abc'
spec = modelsdl.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_attributes=False,
rsa_private_key=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=mock.MagicMock(),
)
ops.output_parameters(go, spec)
assert util.is_not_empty(go.log_file)
spec = modelsul.Specification(upload_options=options.Upload(
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
mode=azmodels.StorageModes.Auto,
one_shot_bytes=0,
overwrite=True,
recursive=True,
rename=False,
rsa_public_key=None,
stdin_as_page_blob_size=0,
store_file_properties=options.FileProperties(
attributes=True,
md5=True,
),
strip_components=0,
vectored_io=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_source_path=mock.MagicMock())
ops.output_parameters(go, spec)
assert util.is_not_empty(go.log_file)
spec = modelssc.Specification(synccopy_options=options.SyncCopy(
delete_extraneous_destination=False,
dest_mode=azmodels.StorageModes.Auto,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
))
ops.output_parameters(go, spec)
assert util.is_not_empty(go.log_file)
def test_downloadspecification():
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
content_type=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
asp = azops.SourcePath()
p = 'some/remote/path'
asp.add_path_with_storage_account(p, 'sa')
ds.add_azure_source_path(asp)
assert ds.options.check_file_md5
assert not ds.skip_on.lmt_ge
assert ds.destination.path == pathlib.Path('dest')
assert len(ds.sources) == 1
assert p in ds.sources[0]._path_map
assert ds.sources[0]._path_map[p] == 'sa'
with pytest.raises(ValueError):
ds = models.Specification(
download_options=options.Download(
check_file_md5=False,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
content_type=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
with pytest.raises(ValueError):
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
max_single_object_concurrency=0,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=False,
cache_control=None,
content_type=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
if util.on_windows():
patch_func = 'time.sleep'
else:
patch_func = 'os.getuid'
with mock.patch(patch_func) as patched_getuid:
patched_getuid.return_value = 1
with pytest.raises(ValueError):
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=-1,
delete_extraneous_destination=False,
delete_only=False,
max_single_object_concurrency=8,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_properties=options.FileProperties(
attributes=True,
cache_control=None,
content_type=None,
lmt=False,
md5=None,
),
rsa_private_key=None,
strip_components=0,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
def test_specification(tmpdir):
lsp = upload.LocalSourcePath()
lsp.add_paths(['-', '/dev/stdin'])
with pytest.raises(ValueError):
upload.Specification(
upload_options=options.Upload(
access_tier=None,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
mode=azmodels.StorageModes.Auto,
one_shot_bytes=0,
overwrite=True,
recursive=True,
rename=True,
rsa_public_key=None,
stdin_as_page_blob_size=0,
store_file_properties=options.FileProperties(
attributes=True,
cache_control='cc',
content_type='ct',
lmt=None,
md5=True,
),
strip_components=0,
vectored_io=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_source_path=lsp,
)
lsp = upload.LocalSourcePath()
lsp.add_path(str(tmpdir))
with pytest.raises(ValueError):
upload.Specification(
upload_options=options.Upload(
access_tier=None,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
mode=azmodels.StorageModes.Auto,
one_shot_bytes=0,
overwrite=True,
recursive=True,
rename=True,
rsa_public_key=None,
stdin_as_page_blob_size=0,
store_file_properties=options.FileProperties(
attributes=True,
cache_control='cc',
content_type='ct',
lmt=None,
md5=True,
),
strip_components=0,
vectored_io=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_source_path=lsp,
)
lsp = upload.LocalSourcePath()
lsp.add_path(str(tmpdir))
with pytest.raises(ValueError):
upload.Specification(
upload_options=options.Upload(
access_tier=None,
chunk_size_bytes=-1,
delete_extraneous_destination=False,
delete_only=False,
mode=azmodels.StorageModes.Auto,
one_shot_bytes=0,
overwrite=True,
recursive=True,
rename=False,
rsa_public_key=None,
stdin_as_page_blob_size=0,
store_file_properties=options.FileProperties(
attributes=True,
cache_control='cc',
content_type='ct',
lmt=None,
md5=True,
),
strip_components=0,
vectored_io=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_source_path=lsp,
)
with pytest.raises(ValueError):
upload.Specification(
upload_options=options.Upload(
access_tier=None,
chunk_size_bytes=upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES + 1,
delete_extraneous_destination=False,
delete_only=False,
mode=azmodels.StorageModes.Auto,
one_shot_bytes=0,
overwrite=True,
recursive=True,
rename=False,
rsa_public_key=None,
stdin_as_page_blob_size=0,
store_file_properties=options.FileProperties(
attributes=True,
cache_control='cc',
content_type='ct',
lmt=None,
md5=True,
),
strip_components=0,
vectored_io=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_source_path=lsp,
)
with pytest.raises(ValueError):
upload.Specification(
upload_options=options.Upload(
access_tier=None,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
mode=azmodels.StorageModes.Auto,
one_shot_bytes=-1,
overwrite=True,
recursive=True,
rename=False,
rsa_public_key=None,
stdin_as_page_blob_size=0,
store_file_properties=options.FileProperties(
attributes=True,
cache_control='cc',
content_type='ct',
lmt=None,
md5=True,
),
strip_components=0,
vectored_io=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_source_path=lsp,
)
with pytest.raises(ValueError):
upload.Specification(
upload_options=options.Upload(
access_tier=None,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
mode=azmodels.StorageModes.Auto,
one_shot_bytes=upload._MAX_BLOCK_BLOB_ONESHOT_BYTES + 1,
overwrite=True,
recursive=True,
rename=False,
rsa_public_key=None,
stdin_as_page_blob_size=0,
store_file_properties=options.FileProperties(
attributes=True,
cache_control=None,
content_type=None,
lmt=None,
md5=True,
),
strip_components=0,
vectored_io=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_source_path=lsp,
)
spec = upload.Specification(
upload_options=options.Upload(
access_tier=None,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
delete_only=False,
mode=azmodels.StorageModes.Auto,
one_shot_bytes=0,
overwrite=True,
recursive=True,
rename=False,
rsa_public_key=None,
stdin_as_page_blob_size=0,
store_file_properties=options.FileProperties(
attributes=True,
cache_control=None,
content_type=None,
lmt=None,
md5=True,
),
strip_components=0,
vectored_io=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_source_path=lsp,
)
spec.add_azure_destination_path(azops.DestinationPath())
assert len(spec.destinations) == 1
def test_downloadspecification():
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_attributes=False,
rsa_private_key=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
asp = azops.SourcePath()
p = 'some/remote/path'
asp.add_path_with_storage_account(p, 'sa')
ds.add_azure_source_path(asp)
assert ds.options.check_file_md5
assert not ds.skip_on.lmt_ge
assert ds.destination.path == pathlib.Path('dest')
assert len(ds.sources) == 1
assert p in ds.sources[0]._path_map
assert ds.sources[0]._path_map[p] == 'sa'
with pytest.raises(ValueError):
ds = models.Specification(
download_options=options.Download(
check_file_md5=False,
chunk_size_bytes=4194304,
delete_extraneous_destination=False,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_attributes=False,
rsa_private_key=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
with pytest.raises(ValueError):
ds = models.Specification(
download_options=options.Download(
check_file_md5=True,
chunk_size_bytes=-1,
delete_extraneous_destination=False,
mode=azmodels.StorageModes.Auto,
overwrite=True,
recursive=True,
rename=False,
restore_file_attributes=True,
rsa_private_key=None,
),
skip_on_options=options.SkipOn(
filesize_match=True,
lmt_ge=False,
md5_match=True,
),
local_destination_path=models.LocalDestinationPath('dest'),
)
