def test_start():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._wait_for_transfer_threads = mock.MagicMock()
u._wait_for_disk_threads = mock.MagicMock()
u._md5_offload = mock.MagicMock()
u._md5_offload.finalize_processes = mock.MagicMock()
u._crypto_offload = mock.MagicMock()
u._crypto_offload.finalize_processes = mock.MagicMock()
u._resume = mock.MagicMock()
u._run = mock.MagicMock()
# test keyboard interrupt
u._run.side_effect = KeyboardInterrupt()
u.start()
assert u._run.call_count == 1
assert u._wait_for_transfer_threads.call_count == 1
assert u._wait_for_disk_threads.call_count == 1
assert u._md5_offload.finalize_processes.call_count == 1
assert u._crypto_offload.finalize_processes.call_count == 1
assert u._resume.close.call_count == 1
# test other exception
u._run.side_effect = RuntimeError()
u.start()
def test_finalize_azure_file(sfmeta, sfmd5):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
ase.mode = azmodels.StorageModes.File
ase.is_encrypted = False
ase.replica_targets = [ase]
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = False
ud = mock.MagicMock()
ud.entity = ase
ud.complete_offset_upload = mock.MagicMock()
ud.local_path = lp
ud.unique_id = 'uid'
ud.must_compute_md5 = True
ud.md5.digest.return_value = b'md5'
ud.requires_non_encrypted_md5_put = True
u._finalize_azure_file(ud, {'a': 0})
assert sfmd5.call_count == 2
assert sfmeta.call_count == 2
def test_post_md5_skip_on_check():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
src = mock.MagicMock()
src.absolute_path = 'abspath'
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
id = ops.Uploader.create_unique_id(src, ase)
u._md5_map[id] = (src, ase)
u._upload_set.add(id)
u._upload_total += 1
u._post_md5_skip_on_check(id, True)
assert len(u._md5_map) == 0
assert id not in u._upload_set
assert u._upload_total == 0
u._md5_map[id] = (src, ase)
u._upload_set.add(id)
u._upload_total += 1
u._add_to_upload_queue = mock.MagicMock()
u._post_md5_skip_on_check(id, False)
assert len(u._md5_map) == 0
assert id in u._upload_set
assert u._upload_total == 1
assert u._add_to_upload_queue.call_count == 1
def test_finalize_block_blob(pbl):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
ase.mode = azmodels.StorageModes.Block
ase.is_encrypted = False
ase.replica_targets = [ase]
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = False
ud = mock.MagicMock()
ud.entity = ase
ud.complete_offset_upload = mock.MagicMock()
ud.local_path = lp
ud.unique_id = 'uid'
ud.must_compute_md5 = True
ud.md5.digest.return_value = b'md5'
u._finalize_block_blob(ud, mock.MagicMock())
assert pbl.call_count == 2
ud.must_compute_md5 = False
ase.replica_targets = []
u._finalize_block_blob(ud, mock.MagicMock())
assert pbl.call_count == 3
def test_prepare_upload(page_cb, cf, capd, cs, append_cb, cc):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
ase.mode = azmodels.StorageModes.Append
ase.append_create = True
u._prepare_upload(ase)
assert cc.call_count == 1
assert append_cb.call_count == 1
ase.mode = azmodels.StorageModes.Block
ase.append_create = False
u._prepare_upload(ase)
assert cc.call_count == 2
ase.mode = azmodels.StorageModes.File
u._prepare_upload(ase)
assert cs.call_count == 1
assert capd.call_count == 1
assert cf.call_count == 1
ase.mode = azmodels.StorageModes.Page
u._prepare_upload(ase)
assert cc.call_count == 3
assert page_cb.call_count == 1
def test_set_blob_properties(sbp):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
ase.mode = azmodels.StorageModes.Block
ase.is_encrypted = False
ase.replica_targets = [ase]
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = False
ud = mock.MagicMock()
ud.entity = ase
ud.complete_offset_upload = mock.MagicMock()
ud.local_path = lp
ud.unique_id = 'uid'
ud.must_compute_md5 = True
ud.md5.digest.return_value = b'md5'
u._set_blob_properties(ud)
assert sbp.call_count == 2
ud.requires_non_encrypted_md5_put = False
ud.must_compute_md5 = False
ase.cache_control = 'cc'
u._set_blob_properties(ud)
assert sbp.call_count == 4
def test_set_blob_metadata(sbm):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
ase.mode = azmodels.StorageModes.Block
ase.is_encrypted = False
ase.replica_targets = [ase]
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = False
ud = mock.MagicMock()
ud.entity = ase
ud.complete_offset_upload = mock.MagicMock()
ud.local_path = lp
ud.unique_id = 'uid'
u._set_blob_metadata(ud, mock.MagicMock())
assert sbm.call_count == 2
def test_check_for_existing_remote(gbp, gfp):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
sa = mock.MagicMock()
sa.name = 'name'
sa.endpoint = 'ep'
u._spec.options.mode = azmodels.StorageModes.File
gfp.return_value = None
assert u._check_for_existing_remote(sa, 'cont', 'name') is None
with mock.patch(
'blobxfer.models.crypto.EncryptionMetadata.'
'encryption_metadata_exists', return_value=False):
gfp.return_value = mock.MagicMock()
assert u._check_for_existing_remote(sa, 'cont', 'name') is not None
with mock.patch(
'blobxfer.models.crypto.EncryptionMetadata.'
'encryption_metadata_exists', return_value=True):
with mock.patch(
'blobxfer.models.crypto.EncryptionMetadata.convert_from_json'):
gfp.return_value = mock.MagicMock()
assert u._check_for_existing_remote(sa, 'cont', 'name') is not None
u._spec.options.mode = azmodels.StorageModes.Block
gbp.return_value = None
assert u._check_for_existing_remote(sa, 'cont', 'name') is None
with mock.patch(
'blobxfer.models.crypto.EncryptionMetadata.'
'encryption_metadata_exists', return_value=False):
gbp.return_value = mock.MagicMock()
assert u._check_for_existing_remote(sa, 'cont', 'name') is not None
def test_check_for_existing_remote(gbp, gfp):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
sa = mock.MagicMock()
sa.name = 'name'
sa.endpoint = 'ep'
u._spec.options.mode = azmodels.StorageModes.File
gfp.return_value = None
assert u._check_for_existing_remote(sa, 'cont', 'name') is None
with mock.patch(
'blobxfer.models.crypto.EncryptionMetadata.'
'encryption_metadata_exists', return_value=False):
gfp.return_value = mock.MagicMock()
assert u._check_for_existing_remote(sa, 'cont', 'name') is not None
with mock.patch(
'blobxfer.models.crypto.EncryptionMetadata.'
'encryption_metadata_exists', return_value=True):
with mock.patch(
'blobxfer.models.crypto.EncryptionMetadata.convert_from_json'):
gfp.return_value = mock.MagicMock()
assert u._check_for_existing_remote(sa, 'cont', 'name') is not None
u._spec.options.mode = azmodels.StorageModes.Block
gbp.return_value = None
assert u._check_for_existing_remote(sa, 'cont', 'name') is None
with mock.patch(
'blobxfer.models.crypto.EncryptionMetadata.'
'encryption_metadata_exists', return_value=False):
gbp.return_value = mock.MagicMock()
assert u._check_for_existing_remote(sa, 'cont', 'name') is not None
# check access tiers
with mock.patch(
'blobxfer.models.crypto.EncryptionMetadata.'
'encryption_metadata_exists', return_value=False):
gbp.return_value = mock.MagicMock()
gbp.return_value.properties.blob_type = \
azure.storage.blob.models._BlobTypes.BlockBlob
gbp.return_value.properties.blob_tier = None
u._spec.options.access_tier = None
ase = u._check_for_existing_remote(sa, 'cont', 'name')
assert ase is not None
assert ase.access_tier is None
gbp.return_value.properties.blob_tier = 'Cool'
ase = u._check_for_existing_remote(sa, 'cont', 'name')
assert ase is not None
assert ase.access_tier is None
u._spec.options.access_tier = 'Hot'
ase = u._check_for_existing_remote(sa, 'cont', 'name')
assert ase is not None
assert ase.access_tier == 'Hot'
def test_put_data(pp, pfr, pb, cb, ab):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
offsets = mock.MagicMock()
offsets.chunk_num = 0
offsets.num_bytes = 1
offsets.range_start = 10
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = True
ud = mock.MagicMock()
ud.entity.mode = azmodels.StorageModes.Append
ud.complete_offset_upload = mock.MagicMock()
ud.local_path = lp
ase.mode = azmodels.StorageModes.Append
u._put_data(ud, ase, offsets, b'\0')
assert ab.call_count == 1
ase.mode = azmodels.StorageModes.Block
ud.is_one_shot_block_blob = True
ud.entity.is_encrypted = False
ud.must_compute_md5 = True
ud.md5.digest.return_value = b'md5'
u._put_data(ud, ase, offsets, b'\0')
assert cb.call_count == 1
ud.must_compute_md5 = False
u._put_data(ud, ase, offsets, b'\0')
assert cb.call_count == 2
ud.is_one_shot_block_blob = False
u._put_data(ud, ase, offsets, b'\0')
assert pb.call_count == 1
ase.mode = azmodels.StorageModes.File
u._put_data(ud, ase, offsets, b'\0')
assert pfr.call_count == 1
ase.mode = azmodels.StorageModes.Page
u._put_data(ud, ase, offsets, None)
assert pp.call_count == 0
ase.mode = azmodels.StorageModes.Page
u._put_data(ud, ase, offsets, b'\0')
assert pp.call_count == 0
ase.mode = azmodels.StorageModes.Page
u._put_data(ud, ase, offsets, b'1')
assert pp.call_count == 1
def test_generate_destination_for_source():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._spec.options.overwrite = False
u._check_for_existing_remote = mock.MagicMock()
lp = mock.MagicMock()
lp.relative_path = pathlib.Path('a/b/c/d')
lp.absolute_path = pathlib.Path('abs/rel/a/b/c/d')
lp.view.fd_start = 0
lp.use_stdin = False
sa = mock.MagicMock()
sa.name = 'name'
sa.endpoint = 'ep'
u._spec.options.strip_components = 1
u._spec.options.rename = True
u._get_destination_paths = mock.MagicMock()
u._get_destination_paths.return_value = [
(sa, 'cont', '', 'dpath'),
]
with pytest.raises(ValueError):
next(u._generate_destination_for_source(lp))
lp.relative_path = pathlib.Path('rel/a')
lp.absolute_path = pathlib.Path('abs/rel/a')
u._spec.options.strip_components = 0
u._spec.options.rename = False
u._get_destination_paths.return_value = [
(sa, 'cont', 'name', 'dpath'),
]
u._spec.options.mode = azmodels.StorageModes.Block
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Stripe
a, b = next(u._generate_destination_for_source(lp))
assert a == sa
assert b is not None
assert u._check_for_existing_remote.call_count == 0
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Disabled
a, b = next(u._generate_destination_for_source(lp))
assert a == sa
assert b is not None
assert u._check_for_existing_remote.call_count == 1
# check no-read permission
sa.can_read_object = False
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Disabled
a, b = next(u._generate_destination_for_source(lp))
assert a == sa
assert b is not None
assert u._check_for_existing_remote.call_count == 2
def test_get_destination_paths():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
paths = mock.MagicMock()
paths.paths = [pathlib.Path('a/b')]
u._spec.destinations = [paths]
sa, cont, dir, dpath = next(u._get_destination_paths())
assert cont == 'a'
assert dir == 'b'
assert dpath == pathlib.Path('a/b')
def test_initialize_transfer_threads():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.concurrency.transfer_threads = 1
try:
u._initialize_transfer_threads()
assert len(u._transfer_threads) == 1
finally:
u._wait_for_transfer_threads(True)
for thr in u._transfer_threads:
assert not thr.is_alive()
def test_update_progress_bar():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
with mock.patch(
'blobxfer.operations.progress.update_progress_bar') as patched_upb:
u._all_files_processed = False
u._update_progress_bar()
assert patched_upb.call_count == 0
u._all_files_processed = True
u._update_progress_bar()
assert patched_upb.call_count == 1
def test_delete_extraneous_files(db, lab, df, laf):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
# test no delete
u._spec.options.delete_extraneous_destination = False
u._get_destination_paths = mock.MagicMock()
u._delete_extraneous_files()
assert u._get_destination_paths.call_count == 0
# test file delete
u._spec.options.delete_extraneous_destination = True
u._spec.options.mode = azmodels.StorageModes.File
sa1 = mock.MagicMock()
sa1.name = 'name'
sa1.endpoint = 'ep'
sa1.file_client.primary_endpoint = 'ep'
u._get_destination_paths = mock.MagicMock()
u._get_destination_paths.return_value = [
(sa1, 'cont', None, None),
(sa1, 'cont', None, None),
]
laf.return_value = ['filename']
u._delete_extraneous_files()
assert laf.call_count == 1
assert df.call_count == 1
# test blob delete
u._spec.options.delete_extraneous_destination = True
u._spec.options.mode = azmodels.StorageModes.Block
sa1 = mock.MagicMock()
sa1.name = 'name'
sa1.endpoint = 'ep'
sa1.block_blob_client.primary_endpoint = 'ep'
u._get_destination_paths = mock.MagicMock()
u._get_destination_paths.return_value = [
(sa1, 'cont', None, None),
]
blob = mock.MagicMock()
blob.name = 'blobname'
lab.return_value = [blob]
u._delete_extraneous_files()
assert lab.call_count == 1
assert db.call_count == 1
def test_start():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._spec.options.overwrite = False
u._spec.options.delete_only = False
u._wait_for_transfer_threads = mock.MagicMock()
u._wait_for_disk_threads = mock.MagicMock()
u._md5_offload = mock.MagicMock()
u._md5_offload.finalize_processes = mock.MagicMock()
u._crypto_offload = mock.MagicMock()
u._crypto_offload.finalize_processes = mock.MagicMock()
u._resume = mock.MagicMock()
u._run = mock.MagicMock()
# test keyboard interrupt
u._run.side_effect = KeyboardInterrupt()
with pytest.raises(KeyboardInterrupt):
u.start()
assert u._run.call_count == 1
assert u._wait_for_transfer_threads.call_count == 1
assert u._wait_for_disk_threads.call_count == 1
assert u._md5_offload.finalize_processes.call_count == 1
assert u._crypto_offload.finalize_processes.call_count == 1
assert u._resume.close.call_count == 1
# test other exception
u._run.side_effect = RuntimeError()
with pytest.raises(RuntimeError):
u.start()
assert u._run.call_count == 2
assert u._wait_for_transfer_threads.call_count == 2
assert u._wait_for_disk_threads.call_count == 2
assert u._md5_offload.finalize_processes.call_count == 2
assert u._crypto_offload.finalize_processes.call_count == 2
assert u._resume.close.call_count == 2
u._run.side_effect = RuntimeError()
with pytest.raises(RuntimeError):
u._wait_for_transfer_threads = mock.MagicMock(
side_effect=RuntimeError('oops'))
u._upload_terminate = True
u.start()
assert u._run.call_count == 3
assert u._wait_for_transfer_threads.call_count == 1
assert u._wait_for_disk_threads.call_count == 2
assert u._md5_offload.finalize_processes.call_count == 3
assert u._crypto_offload.finalize_processes.call_count == 3
assert u._resume.close.call_count == 3
def test_pre_md5_skip_on_check():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
src = mock.MagicMock()
src.absolute_path = 'abspath'
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
u._md5_offload = mock.MagicMock()
u._pre_md5_skip_on_check(src, ase)
assert len(u._md5_map) == 1
assert u._md5_offload.add_localfile_for_md5_check.call_count == 1
def test_worker_thread_transfer():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._transfer_queue.put(
(mock.MagicMock, mock.MagicMock, mock.MagicMock, mock.MagicMock))
u._transfer_queue.put(
(mock.MagicMock, mock.MagicMock, mock.MagicMock, mock.MagicMock))
u._process_transfer = mock.MagicMock()
u._process_transfer.side_effect = [None, Exception()]
with mock.patch('blobxfer.operations.upload.Uploader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
patched_tc.side_effect = [False, False, True]
u._worker_thread_transfer()
assert u._process_transfer.call_count == 2
assert len(u._exceptions) == 1
def test_check_for_uploads_from_md5():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._md5_offload = mock.MagicMock()
u._post_md5_skip_on_check = mock.MagicMock()
with mock.patch(
'blobxfer.operations.upload.Uploader.termination_check_md5',
new_callable=mock.PropertyMock) as patched_tcm:
patched_tcm.side_effect = [False, False, False, True, True]
u._md5_offload.pop_done_queue.side_effect = [
None, mock.MagicMock(), None
]
u._check_for_uploads_from_md5()
assert u._post_md5_skip_on_check.call_count == 1
def test_worker_thread_upload(ts):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.concurrency.transfer_threads = 1
u._transfer_set = mock.MagicMock()
u._transfer_set.__len__.side_effect = [5, 0, 0, 0]
u._upload_queue.put(mock.MagicMock)
u._upload_queue.put(mock.MagicMock)
u._process_upload_descriptor = mock.MagicMock()
u._process_upload_descriptor.side_effect = [None, Exception()]
with mock.patch('blobxfer.operations.upload.Uploader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
patched_tc.side_effect = [False, False, False, False, True]
u._worker_thread_upload()
assert u._process_upload_descriptor.call_count == 2
assert len(u._exceptions) == 1
def test_process_transfer():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._put_data = mock.MagicMock()
u._update_progress_bar = mock.MagicMock()
offsets = mock.MagicMock()
offsets.chunk_num = 0
offsets.num_bytes = 1
offsets.range_start = 10
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = True
ud = mock.MagicMock()
ud.entity.mode = azmodels.StorageModes.Append
ud.complete_offset_upload = mock.MagicMock()
ud.local_path = lp
id = ops.Uploader.create_unique_transfer_id(lp, ase, offsets)
u._transfer_set.add(id)
u._process_transfer(ud, ase, offsets, mock.MagicMock())
assert u._upload_bytes_total == 1
assert u._upload_bytes_sofar == 1
assert len(u._transfer_set) == 0
assert ud.complete_offset_upload.call_count == 1
assert u._upload_queue.qsize() == 1
assert u._update_progress_bar.call_count == 1
lp.use_stdin = False
u._transfer_set.add(id)
u._process_transfer(ud, ase, offsets, mock.MagicMock())
assert u._upload_bytes_total == 11
assert u._upload_bytes_sofar == 2
assert len(u._transfer_set) == 0
assert ud.complete_offset_upload.call_count == 2
assert u._upload_queue.qsize() == 2
assert u._update_progress_bar.call_count == 2
def test_add_to_upload_queue():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._spec.options.chunk_size_bytes = 32
src = mock.MagicMock()
src.absolute_path = 'abspath'
src.size = 32
src.use_stdin = False
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.encryption_metadata.symmetric_key = 'abc'
id = ops.Uploader.create_unique_id(src, ase)
u._add_to_upload_queue(src, ase, id)
assert len(u._ud_map) == 1
assert u._upload_queue.qsize() == 1
assert u._upload_start_time is not None
def test_finalize_upload():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
ase.mode = azmodels.StorageModes.Block
ase.is_encrypted = False
ase.replica_targets = [ase]
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = False
ud = mock.MagicMock()
ud.entity = ase
ud.complete_offset_upload = mock.MagicMock()
ud.local_path = lp
ud.unique_id = 'uid'
ud.requires_put_block_list = True
u._finalize_block_blob = mock.MagicMock()
u._finalize_upload(ud)
assert u._finalize_block_blob.call_count == 1
ud.requires_put_block_list = False
ud.remote_is_page_blob = True
u._finalize_nonblock_blob = mock.MagicMock()
u._finalize_upload(ud)
assert u._finalize_nonblock_blob.call_count == 1
ud.remote_is_page_blob = False
ud.remote_is_append_blob = False
ud.remote_is_file = True
u._finalize_azure_file = mock.MagicMock()
u._finalize_upload(ud)
assert u._finalize_azure_file.call_count == 1
def test_resize_blob(rb):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
ase.mode = azmodels.StorageModes.Block
ase.is_encrypted = False
ase.replica_targets = [ase]
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = True
ud = mock.MagicMock()
ud.entity = ase
ud.local_path = lp
ud.unique_id = 'uid'
u._resize_blob(ud, 512)
assert rb.call_count == 2
def test_finalize_nonblock_blob():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
ase.mode = azmodels.StorageModes.Block
ase.is_encrypted = False
ase.replica_targets = [ase]
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = False
ud = mock.MagicMock()
ud.entity = ase
ud.local_path = lp
ud.unique_id = 'uid'
ud.requires_non_encrypted_md5_put = True
ud.requires_resize.return_value = (False, None)
u._set_blob_properties = mock.MagicMock()
u._set_blob_metadata = mock.MagicMock()
u._resize_blob = mock.MagicMock()
u._finalize_nonblock_blob(ud, {'a': 0})
assert u._set_blob_properties.call_count == 1
assert u._set_blob_metadata.call_count == 1
assert u._resize_blob.call_count == 0
# resize required
ud.requires_resize.return_value = (True, 512)
u._finalize_nonblock_blob(ud, {'a': 0})
assert u._resize_blob.call_count == 1
def test_check_upload_conditions(gmfm):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
ase.mode = azmodels.StorageModes.Block
ase.append_create = True
ase.is_encrypted = False
ase.from_local = False
lp = mock.MagicMock()
lp.absolute_path = pathlib.Path('lpabspath')
lp.view.fd_start = 0
lp.use_stdin = False
assert u._check_upload_conditions(lp, ase) == ops.UploadAction.Skip
lp.use_stdin = True
assert u._check_upload_conditions(lp, None) == ops.UploadAction.Upload
u._spec.options.overwrite = False
ase.mode = azmodels.StorageModes.Append
assert u._check_upload_conditions(lp, ase) == ops.UploadAction.Upload
assert not ase.append_create
ase.mode = azmodels.StorageModes.Block
ase.append_create = True
assert u._check_upload_conditions(lp, ase) == ops.UploadAction.Skip
assert ase.append_create
u._spec.options.overwrite = True
u._spec.skip_on.md5_match = True
gmfm.return_value = 'md5'
assert u._check_upload_conditions(lp, ase) == ops.UploadAction.CheckMd5
u._spec.skip_on.md5_match = False
u._spec.skip_on.filesize_match = False
u._spec.skip_on.lmt_ge = False
assert u._check_upload_conditions(lp, ase) == ops.UploadAction.Upload
# size mismatch, page
u._spec.skip_on.filesize_match = True
ase.mode = azmodels.StorageModes.Page
lp.size = 1
assert u._check_upload_conditions(lp, ase) == ops.UploadAction.Upload
# size match
u._spec.skip_on.filesize_match = True
ase.mode = azmodels.StorageModes.Block
lp.size = ase.size
assert u._check_upload_conditions(lp, ase) == ops.UploadAction.Skip
# lmt match
u._spec.skip_on.filesize_match = False
u._spec.skip_on.lmt_ge = True
ase.lmt = 0
with mock.patch('blobxfer.util.datetime_from_timestamp') as patched_dft:
patched_dft.return_value = 0
assert u._check_upload_conditions(lp, ase) == ops.UploadAction.Skip
# lmt mismatch
u._spec.skip_on.lmt_ge = True
ase.lmt = 0
with mock.patch('blobxfer.util.datetime_from_timestamp') as patched_dft:
patched_dft.return_value = 1
assert u._check_upload_conditions(lp, ase) == ops.UploadAction.Upload
def test_delete_extraneous_files(db, lab, df, laf):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
# test no delete
u._spec.options.delete_extraneous_destination = False
u._get_destination_paths = mock.MagicMock()
u._delete_extraneous_files()
assert u._get_destination_paths.call_count == 0
# test file delete
u._spec.options.delete_extraneous_destination = True
u._spec.options.mode = azmodels.StorageModes.File
sa1 = mock.MagicMock()
sa1.name = 'name'
sa1.endpoint = 'ep'
sa1.file_client.primary_endpoint = 'ep'
laf.return_value = ['filename']
# test relative path failure
u._get_destination_paths = mock.MagicMock()
u._get_destination_paths.return_value = [
(sa1, 'cont', 'vpath', ''),
]
u._delete_extraneous_files()
assert laf.call_count == 1
assert df.call_count == 0
# test actual delete
u._get_destination_paths = mock.MagicMock()
u._get_destination_paths.return_value = [
(sa1, 'cont', '', ''),
(sa1, 'cont', '', ''),
]
u._general_options.dry_run = True
u._delete_extraneous_files()
assert laf.call_count == 2
assert df.call_count == 0
u._general_options.dry_run = False
u._delete_extraneous_files()
assert laf.call_count == 3
assert df.call_count == 1
# test blob delete
u._spec.options.delete_extraneous_destination = True
u._spec.options.mode = azmodels.StorageModes.Block
sa1 = mock.MagicMock()
sa1.name = 'name'
sa1.endpoint = 'ep'
sa1.block_blob_client.primary_endpoint = 'ep'
blob = mock.MagicMock()
blob.name = 'blobname'
lab.return_value = [blob]
# test relative path failure
u._get_destination_paths = mock.MagicMock()
u._get_destination_paths.return_value = [
(sa1, 'cont', 'vpath', ''),
]
u._delete_extraneous_files()
assert lab.call_count == 1
assert db.call_count == 0
# test actual delete
u._get_destination_paths = mock.MagicMock()
u._get_destination_paths.return_value = [
(sa1, 'cont', '', ''),
]
u._general_options.dry_run = True
u._delete_extraneous_files()
assert lab.call_count == 2
assert db.call_count == 0
u._general_options.dry_run = False
u._delete_extraneous_files()
assert lab.call_count == 3
assert db.call_count == 1
def test_process_upload_descriptor():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
ase = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.size = 10
ase.mode = azmodels.StorageModes.Block
ase.is_encrypted = True
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = False
ud = mock.MagicMock()
ud.entity = ase
ud.complete_offset_upload = mock.MagicMock()
ud.local_path = lp
ud.next_offsets.return_value = (None, 1)
ud.all_operations_completed = True
ud.unique_id = 'uid'
u._finalize_upload = mock.MagicMock()
u._ud_map['uid'] = 0
u._upload_set.add('uid')
# test resume and completed
u._process_upload_descriptor(ud)
assert u._upload_bytes_total == 10
assert u._upload_bytes_sofar == 1
assert u._finalize_upload.call_count == 1
assert len(u._ud_map) == 0
assert len(u._upload_set) == 0
assert u._upload_sofar == 1
# test nothing
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
ud.all_operations_completed = False
ud.next_offsets.return_value = (None, None)
u._process_upload_descriptor(ud)
assert u._upload_queue.qsize() == 1
# test encrypted
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
offsets = mock.MagicMock()
offsets.chunk_num = 0
offsets.num_bytes = 1
offsets.range_start = 10
ud.next_offsets.return_value = (offsets, None)
u._prepare_upload = mock.MagicMock()
ase2 = mock.MagicMock()
ase2._client.primary_endpoint = 'ep'
ase2.path = 'asepath2'
ase2.size = 10
ase2.mode = azmodels.StorageModes.Block
ase.replica_targets = [ase2]
ase.is_encrypted = True
ud.read_data.return_value = (b'\0', None)
with mock.patch('blobxfer.operations.crypto.aes_cbc_encrypt_data',
return_value=b'\0' * 16):
u._process_upload_descriptor(ud)
assert u._upload_queue.qsize() == 1
assert u._prepare_upload.call_count == 2
assert ud.hmac_data.call_count == 2
assert u._transfer_queue.qsize() == 2
assert len(u._transfer_set) == 2
# test stdin
ase.is_encrypted = False
lp = mock.MagicMock()
lp.absolute_path = 'lpabspath'
lp.view.fd_start = 0
lp.use_stdin = True
ud.local_path = lp
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._prepare_upload = mock.MagicMock()
ud.read_data.return_value = (False, offsets)
u._process_upload_descriptor(ud)
assert u._upload_queue.qsize() == 1
assert u._transfer_queue.qsize() == 0
assert len(u._transfer_set) == 0
def test_termination_check():
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
assert not u.termination_check
assert not u.termination_check_md5
def test_run(lfmo, urm, tmpdir):
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._initialize_disk_threads = mock.MagicMock()
u._initialize_transfer_threads = mock.MagicMock()
u._general_options.concurrency.disk_threads = 1
u._general_options.concurrency.transfer_threads = 1
u._general_options.concurrency.md5_processes = 1
u._general_options.concurrency.crypto_processes = 1
u._general_options.resume_file = 'resume'
u._spec.options.store_file_properties.md5 = True
u._spec.skip_on.md5_match = True
u._spec.options.rsa_public_key = 'abc'
u._spec.options.chunk_size_bytes = 0
u._spec.options.one_shot_bytes = 0
u._spec.options.delete_only = False
# check rename failure
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = True
with pytest.raises(RuntimeError):
u._run()
u._upload_terminate = True
assert urm.call_count == 0
assert lfmo.call_count == 0
assert lfmo.initialize_check_thread.call_count == 0
assert u._initialize_disk_threads.call_count == 0
assert u._initialize_transfer_threads.call_count == 0
# check dupe
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = False
ase = mock.MagicMock()
ase.client.primary_endpoint = 'ep'
ase._client.primary_endpoint = 'ep'
ase.path = 'asepath'
ase.mode = azmodels.StorageModes.Block
ase.is_encrypted = False
ase.replica_targets = None
ase.container = 'cont'
ase.name = 'name'
ase.size = 10
ase2 = mock.MagicMock()
ase2.client.primary_endpoint = 'ep2'
ase2._client.primary_endpoint = 'ep2'
ase2.path = 'asepath2'
ase2.mode = azmodels.StorageModes.Block
ase2.is_encrypted = False
ase2.replica_targets = None
ase2.container = 'cont2'
ase2.name = 'name2'
ase2.size = 10
sa = mock.MagicMock()
sa.name = 'name'
sa.endpoint = 'ep'
sa.block_blob_client.primary_endpoint = 'pep'
tmpdir.join('a').write('z' * 10)
lp = mock.MagicMock()
lp.relative_path = pathlib.Path('a')
lp.absolute_path = pathlib.Path(str(tmpdir.join('a')))
lp.view.fd_start = 0
lp.view.fd_end = 10
lp.use_stdin = False
lp.size = 10
lp.total_size = 10
u._generate_destination_for_source = mock.MagicMock()
with pytest.raises(RuntimeError):
u._generate_destination_for_source.return_value = [(sa, ase),
(sa, ase)]
u._spec.sources.files.return_value = [lp]
u._run()
u._upload_terminate = True
assert urm.call_count == 1
assert lfmo.call_count == 1
assert u._md5_offload.initialize_check_thread.call_count == 1
assert u._initialize_disk_threads.call_count == 1
assert u._initialize_transfer_threads.call_count == 1
# mismatch exception raise
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Disabled
u._check_upload_conditions = mock.MagicMock()
u._check_upload_conditions.return_value = ops.UploadAction.Skip
u._generate_destination_for_source.return_value = [(sa, ase)]
u._spec.sources.files.return_value = [lp]
with pytest.raises(RuntimeError):
u._run()
u._upload_terminate = True
u._check_upload_conditions.return_value = ops.UploadAction.CheckMd5
with pytest.raises(RuntimeError):
u._run()
u._upload_terminate = True
# regular execution
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._general_options.concurrency.disk_threads = 1
u._general_options.concurrency.transfer_threads = 1
u._general_options.concurrency.md5_processes = 1
u._general_options.concurrency.crypto_processes = 0
u._general_options.resume_file = 'resume'
u._spec.options.store_file_properties.md5 = True
u._spec.skip_on.md5_match = True
u._spec.options.rsa_public_key = None
u._spec.options.chunk_size_bytes = 0
u._spec.options.one_shot_bytes = 0
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = False
u._spec.options.delete_only = False
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Replica
u._check_upload_conditions = mock.MagicMock()
u._check_upload_conditions.return_value = ops.UploadAction.Upload
u._generate_destination_for_source = mock.MagicMock()
u._generate_destination_for_source.return_value = [(sa, ase), (sa, ase2)]
u._spec.sources.files.return_value = [lp]
u._put_data = mock.MagicMock()
u._finalize_upload = mock.MagicMock()
u._upload_start_time = (util.datetime_now() -
datetime.timedelta(seconds=1))
u._run()
assert u._finalize_upload.call_count == 1
# regular execution, skip dry run
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = True
u._general_options.concurrency.disk_threads = 1
u._general_options.concurrency.transfer_threads = 1
u._general_options.concurrency.md5_processes = 1
u._general_options.concurrency.crypto_processes = 0
u._general_options.resume_file = 'resume'
u._spec.options.store_file_properties.md5 = True
u._spec.skip_on.md5_match = True
u._spec.options.rsa_public_key = None
u._spec.options.chunk_size_bytes = 0
u._spec.options.one_shot_bytes = 0
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = False
u._spec.options.delete_only = False
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Replica
u._check_upload_conditions = mock.MagicMock()
u._check_upload_conditions.return_value = ops.UploadAction.Skip
u._generate_destination_for_source = mock.MagicMock()
u._generate_destination_for_source.return_value = [(sa, ase), (sa, ase2)]
u._spec.sources.files.return_value = [lp]
u._put_data = mock.MagicMock()
u._finalize_upload = mock.MagicMock()
u._upload_start_time = (util.datetime_now() -
datetime.timedelta(seconds=1))
u._run()
assert u._finalize_upload.call_count == 0
# delete only
u._spec.options.delete_extraneous_destination = True
u._spec.options.delete_only = True
u._run()
assert u._finalize_upload.call_count == 0
# regular execution, upload dry run
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = True
u._general_options.concurrency.disk_threads = 1
u._general_options.concurrency.transfer_threads = 1
u._general_options.concurrency.md5_processes = 1
u._general_options.concurrency.crypto_processes = 0
u._general_options.resume_file = 'resume'
u._spec.options.store_file_properties.md5 = True
u._spec.skip_on.md5_match = True
u._spec.options.rsa_public_key = None
u._spec.options.chunk_size_bytes = 0
u._spec.options.one_shot_bytes = 0
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = False
u._spec.options.delete_only = False
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Replica
u._check_upload_conditions = mock.MagicMock()
u._check_upload_conditions.return_value = ops.UploadAction.Upload
u._generate_destination_for_source = mock.MagicMock()
u._generate_destination_for_source.return_value = [(sa, ase), (sa, ase2)]
u._spec.sources.files.return_value = [lp]
u._put_data = mock.MagicMock()
u._finalize_upload = mock.MagicMock()
u._upload_start_time = (util.datetime_now() -
datetime.timedelta(seconds=1))
u._run()
assert u._finalize_upload.call_count == 0
# exception raise
u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
u._general_options.dry_run = False
u._general_options.concurrency.disk_threads = 1
u._general_options.concurrency.transfer_threads = 1
u._general_options.concurrency.md5_processes = 1
u._general_options.concurrency.crypto_processes = 0
u._general_options.resume_file = 'resume'
u._spec.options.store_file_properties.md5 = True
u._spec.skip_on.md5_match = True
u._spec.options.rsa_public_key = None
u._spec.options.chunk_size_bytes = 0
u._spec.options.one_shot_bytes = 0
u._spec.sources.can_rename.return_value = False
u._spec.options.rename = False
u._spec.options.delete_only = False
u._spec.options.vectored_io.distribution_mode = \
models.VectoredIoDistributionMode.Disabled
u._check_upload_conditions = mock.MagicMock()
u._check_upload_conditions.return_value = ops.UploadAction.Upload
u._generate_destination_for_source = mock.MagicMock()
u._generate_destination_for_source.return_value = [(sa, ase)]
u._spec.sources.files.return_value = [lp]
with pytest.raises(RuntimeError):
u._process_upload_descriptor = mock.MagicMock()
u._process_upload_descriptor.side_effect = RuntimeError()
u._run()
u._upload_terminate = True