def _create_downloader_for_start(td):
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._cleanup_temporary_files = mock.MagicMock()
d._download_start = datetime.datetime.now(tz=dateutil.tz.tzlocal())
d._initialize_transfer_threads = mock.MagicMock()
d._general_options.concurrency.crypto_processes = 1
d._general_options.concurrency.md5_processes = 1
d._general_options.concurrency.disk_threads = 1
d._general_options.concurrency.transfer_threads = 1
d._general_options.resume_file = pathlib.Path(str(td.join('rf')))
d._spec.sources = []
d._spec.options = mock.MagicMock()
d._spec.options.chunk_size_bytes = 1
d._spec.options.mode = azmodels.StorageModes.Auto
d._spec.options.overwrite = True
d._spec.options.rename = False
d._spec.options.strip_components = 0
d._spec.skip_on = mock.MagicMock()
d._spec.skip_on.md5_match = False
d._spec.skip_on.lmt_ge = False
d._spec.skip_on.filesize_match = False
d._spec.destination = mock.MagicMock()
d._spec.destination.path = pathlib.Path(str(td))
d._download_start_time = util.datetime_now()
d._pre_md5_skip_on_check = mock.MagicMock()
d._check_download_conditions = mock.MagicMock()
d._all_remote_files_processed = False
p = '/cont/remote/path'
asp = azops.SourcePath()
asp.add_path_with_storage_account(p, 'sa')
d._spec.sources.append(asp)
return d
def test_run(srm, gbr, gfr):
s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
s._general_options.dry_run = False
s._general_options.concurrency.transfer_threads = 1
s._general_options.resume_file = 'resume'
s._spec.options.chunk_size_bytes = 0
src_ase = mock.MagicMock()
src_ase._client.primary_endpoint = 'ep'
src_ase.container = 'srccont'
src_ase.path = 'srcasepath'
src_ase.mode = azmodels.StorageModes.Block
src_ase.size = 10
src_ase.name = 'srcase'
dst_ase = mock.MagicMock()
dst_ase._client.primary_endpoint = 'ep2'
dst_ase.container = 'dstcont'
dst_ase.path = 'dstasepath'
dst_ase.mode = azmodels.StorageModes.Block
dst_ase.size = 10
dst_ase.name = 'dstase'
dst_ase.from_local = False
dst_ase.replica_targets = None
s._bind_sources_to_destination = mock.MagicMock()
s._bind_sources_to_destination.return_value = [(src_ase, dst_ase)]
s._prepare_upload = mock.MagicMock()
s._put_data = mock.MagicMock()
s._finalize_upload = mock.MagicMock()
# normal execution
s._synccopy_start_time = (util.datetime_now() -
datetime.timedelta(seconds=1))
s._run()
assert s._prepare_upload.call_count == 1
assert s._put_data.call_count == 1
# dry run
s._general_options.dry_run = True
s._run()
assert s._prepare_upload.call_count == 1
assert s._put_data.call_count == 1
s._general_options.dry_run = False
# replica targets with mismatch
s._synccopy_start_time = None
dst_ase.replica_targets = [dst_ase]
with pytest.raises(RuntimeError):
s._run()
# exception during worker thread
dst_ase.replica_targets = None
with pytest.raises(RuntimeError):
s._process_synccopy_descriptor = mock.MagicMock()
s._process_synccopy_descriptor.side_effect = RuntimeError()
s._run()
def test_update_progress_bar():
go = mock.MagicMock()
go.progress_bar = True
go.log_file = 'abc'
start = util.datetime_now()
ops.update_progress_bar(
go, 'download', start, None, 1, None, 1)
ops.update_progress_bar(
go, 'upload', start, 1, 0, 256, 0, stdin_upload=True)
with mock.patch('blobxfer.util.datetime_now') as patched_dt:
patched_dt.return_value = start
ops.update_progress_bar(
go, 'synccopy', start, 1, 1, 1, 1)
assert util.is_not_empty(go.log_file)
def test_restore_file_lmt(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
lp.touch(mode=0o666, exist_ok=False)
lp.exists()
ts = util.datetime_now() - datetime.timedelta(seconds=60)
ts_posix = time.mktime(ts.timetuple())
stat = lp.stat()
assert stat.st_mtime != ts_posix
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
opts.restore_file_properties.lmt = True
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._lmt = ts
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._restore_file_lmt()
stat = lp.stat()
assert stat.st_mtime == ts_posix
def test_run(lfmo, urm, tmpdir):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._initialize_disk_threads = mock.MagicMock()
u._initialize_transfer_threads = mock.MagicMock()
u._general_options.concurrency.disk_threads = 1
u._general_options.concurrency.transfer_threads = 1
u._general_options.concurrency.md5_processes = 1
u._general_options.concurrency.crypto_processes = 1
u._general_options.resume_file = 'resume'
u._spec.options.store_file_properties.md5 = True
u._spec.skip_on.md5_match = True
u._spec.options.rsa_public_key = 'abc'
u._spec.options.chunk_size_bytes = 0
u._spec.options.one_shot_bytes = 0
u._spec.options.delete_only = False
# check rename failure
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = True
with pytest.raises(RuntimeError):
u._run()
u._upload_terminate = True
assert urm.call_count == 0
assert lfmo.call_count == 0
assert lfmo.initialize_check_thread.call_count == 0
assert u._initialize_disk_threads.call_count == 0
assert u._initialize_transfer_threads.call_count == 0
# check dupe
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = False
ase = mock.MagicMock()
ase.client.primary_endpoint = 'ep'
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.mode = azmodels.StorageModes.Block
ase.is_encrypted = False
ase.replica_targets = None
ase.container = 'cont'
ase.name = 'name'
ase.size = 10
ase2 = mock.MagicMock()
ase2.client.primary_endpoint = 'ep2'
ase2._client.primary_endpoint = 'ep2'
ase2.path = 'asepath2'
ase2.mode = azmodels.StorageModes.Block
ase2.is_encrypted = False
ase2.replica_targets = None
ase2.container = 'cont2'
ase2.name = 'name2'
ase2.size = 10
sa = mock.MagicMock()
sa.name = 'name'
sa.endpoint = 'ep'
sa.block_blob_client.primary_endpoint = 'pep'
tmpdir.join('a').write('z' * 10)
lp = mock.MagicMock()
lp.relative_path = pathlib.Path('a')
lp.absolute_path = pathlib.Path(str(tmpdir.join('a')))
lp.view.fd_start = 0
lp.view.fd_end = 10
lp.use_stdin = False
lp.size = 10
lp.total_size = 10
u._generate_destination_for_source = mock.MagicMock()
with pytest.raises(RuntimeError):
u._generate_destination_for_source.return_value = [(sa, ase),
(sa, ase)]
u._spec.sources.files.return_value = [lp]
u._run()
u._upload_terminate = True
assert urm.call_count == 1
assert lfmo.call_count == 1
assert u._md5_offload.initialize_check_thread.call_count == 1
assert u._initialize_disk_threads.call_count == 1
assert u._initialize_transfer_threads.call_count == 1
# mismatch exception raise
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Disabled
u._check_upload_conditions = mock.MagicMock()
u._check_upload_conditions.return_value = ops.UploadAction.Skip
u._generate_destination_for_source.return_value = [(sa, ase)]
u._spec.sources.files.return_value = [lp]
with pytest.raises(RuntimeError):
u._run()
u._upload_terminate = True
u._check_upload_conditions.return_value = ops.UploadAction.CheckMd5
with pytest.raises(RuntimeError):
u._run()
u._upload_terminate = True
# regular execution
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._general_options.concurrency.disk_threads = 1
u._general_options.concurrency.transfer_threads = 1
u._general_options.concurrency.md5_processes = 1
u._general_options.concurrency.crypto_processes = 0
u._general_options.resume_file = 'resume'
u._spec.options.store_file_properties.md5 = True
u._spec.skip_on.md5_match = True
u._spec.options.rsa_public_key = None
u._spec.options.chunk_size_bytes = 0
u._spec.options.one_shot_bytes = 0
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = False
u._spec.options.delete_only = False
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Replica
u._check_upload_conditions = mock.MagicMock()
u._check_upload_conditions.return_value = ops.UploadAction.Upload
u._generate_destination_for_source = mock.MagicMock()
u._generate_destination_for_source.return_value = [(sa, ase), (sa, ase2)]
u._spec.sources.files.return_value = [lp]
u._put_data = mock.MagicMock()
u._finalize_upload = mock.MagicMock()
u._upload_start_time = (util.datetime_now() -
datetime.timedelta(seconds=1))
u._run()
assert u._finalize_upload.call_count == 1
# regular execution, skip dry run
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = True
u._general_options.concurrency.disk_threads = 1
u._general_options.concurrency.transfer_threads = 1
u._general_options.concurrency.md5_processes = 1
u._general_options.concurrency.crypto_processes = 0
u._general_options.resume_file = 'resume'
u._spec.options.store_file_properties.md5 = True
u._spec.skip_on.md5_match = True
u._spec.options.rsa_public_key = None
u._spec.options.chunk_size_bytes = 0
u._spec.options.one_shot_bytes = 0
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = False
u._spec.options.delete_only = False
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Replica
u._check_upload_conditions = mock.MagicMock()
u._check_upload_conditions.return_value = ops.UploadAction.Skip
u._generate_destination_for_source = mock.MagicMock()
u._generate_destination_for_source.return_value = [(sa, ase), (sa, ase2)]
u._spec.sources.files.return_value = [lp]
u._put_data = mock.MagicMock()
u._finalize_upload = mock.MagicMock()
u._upload_start_time = (util.datetime_now() -
datetime.timedelta(seconds=1))
u._run()
assert u._finalize_upload.call_count == 0
# delete only
u._spec.options.delete_extraneous_destination = True
u._spec.options.delete_only = True
u._run()
assert u._finalize_upload.call_count == 0
# regular execution, upload dry run
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = True
u._general_options.concurrency.disk_threads = 1
u._general_options.concurrency.transfer_threads = 1
u._general_options.concurrency.md5_processes = 1
u._general_options.concurrency.crypto_processes = 0
u._general_options.resume_file = 'resume'
u._spec.options.store_file_properties.md5 = True
u._spec.skip_on.md5_match = True
u._spec.options.rsa_public_key = None
u._spec.options.chunk_size_bytes = 0
u._spec.options.one_shot_bytes = 0
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = False
u._spec.options.delete_only = False
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Replica
u._check_upload_conditions = mock.MagicMock()
u._check_upload_conditions.return_value = ops.UploadAction.Upload
u._generate_destination_for_source = mock.MagicMock()
u._generate_destination_for_source.return_value = [(sa, ase), (sa, ase2)]
u._spec.sources.files.return_value = [lp]
u._put_data = mock.MagicMock()
u._finalize_upload = mock.MagicMock()
u._upload_start_time = (util.datetime_now() -
datetime.timedelta(seconds=1))
u._run()
assert u._finalize_upload.call_count == 0
# exception raise
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._general_options.concurrency.disk_threads = 1
u._general_options.concurrency.transfer_threads = 1
u._general_options.concurrency.md5_processes = 1
u._general_options.concurrency.crypto_processes = 0
u._general_options.resume_file = 'resume'
u._spec.options.store_file_properties.md5 = True
u._spec.skip_on.md5_match = True
u._spec.options.rsa_public_key = None
u._spec.options.chunk_size_bytes = 0
u._spec.options.one_shot_bytes = 0
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = False
u._spec.options.delete_only = False
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Disabled
u._check_upload_conditions = mock.MagicMock()
u._check_upload_conditions.return_value = ops.UploadAction.Upload
u._generate_destination_for_source = mock.MagicMock()
u._generate_destination_for_source.return_value = [(sa, ase)]
u._spec.sources.files.return_value = [lp]
with pytest.raises(RuntimeError):
u._process_upload_descriptor = mock.MagicMock()
u._process_upload_descriptor.side_effect = RuntimeError()
u._run()
u._upload_terminate = True
