def test_descriptor_complete_offset_upload():
opts = mock.MagicMock()
opts.dest_mode = azmodels.StorageModes.Auto
opts.mode = azmodels.StorageModes.Auto
src_ase = azmodels.StorageEntity('cont')
src_ase._mode = azmodels.StorageModes.Block
src_ase._name = 'name'
src_ase._size = 32
src_ase._encryption = None
dst_ase = azmodels.StorageEntity('cont2')
dst_ase._mode = azmodels.StorageModes.Block
dst_ase._name = 'name'
dst_ase._size = 32
dst_ase._encryption = None
dst_ase.replica_targets = [mock.MagicMock()]
d = synccopy.Descriptor(src_ase, dst_ase, None, opts, mock.MagicMock())
d.complete_offset_upload(0)
assert d._outstanding_ops == 1
d.complete_offset_upload(0)
assert d._outstanding_ops == 0
assert 0 not in d._replica_counters
def test_descriptor_compute_chunk_size():
opts = mock.MagicMock()
opts.dest_mode = azmodels.StorageModes.Auto
opts.mode = azmodels.StorageModes.Auto
src_ase = azmodels.StorageEntity('cont')
src_ase._mode = azmodels.StorageModes.Block
src_ase._name = 'name'
src_ase._size = 32
src_ase._encryption = None
dst_ase = azmodels.StorageEntity('cont2')
dst_ase._mode = azmodels.StorageModes.Block
dst_ase._name = 'name'
dst_ase._size = 32
dst_ase._encryption = None
dst_ase.replica_targets = [mock.MagicMock()]
d = synccopy.Descriptor(src_ase, dst_ase, None, opts, mock.MagicMock())
assert d._compute_chunk_size() == \
synccopy._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES
d = synccopy.Descriptor(src_ase, dst_ase, [], opts, mock.MagicMock())
assert d._compute_chunk_size() == d.src_entity.size
b = mock.MagicMock()
b.size = 1
d = synccopy.Descriptor(src_ase, dst_ase, [b], opts, mock.MagicMock())
assert d._compute_chunk_size() == 1
d = synccopy.Descriptor(src_ase, dst_ase, [b, b], opts, mock.MagicMock())
assert d._compute_chunk_size() == -1
def test_descriptor():
opts = mock.MagicMock()
opts.dest_mode = azmodels.StorageModes.Auto
opts.mode = azmodels.StorageModes.Auto
src_ase = azmodels.StorageEntity('cont')
src_ase._mode = azmodels.StorageModes.Block
src_ase._name = 'name'
src_ase._size = 32
src_ase._encryption = None
dst_ase = azmodels.StorageEntity('cont2')
dst_ase._mode = azmodels.StorageModes.Block
dst_ase._name = 'name'
dst_ase._size = 32
dst_ase._encryption = None
dst_ase.replica_targets = [mock.MagicMock()]
d = synccopy.Descriptor(src_ase, dst_ase, None, opts, mock.MagicMock())
assert d._offset == 0
assert d._chunk_num == 0
assert not d._finalized
assert d._src_block_list is None
assert d.src_entity == src_ase
assert d.dst_entity == dst_ase
assert not d.all_operations_completed
assert d.is_resumable
assert d.last_block_num == -1
assert not d.remote_is_file
assert not d.remote_is_page_blob
assert not d.remote_is_append_blob
assert d.is_one_shot_block_blob
assert not d.requires_put_block_list
def test_descriptor(tmpdir):
size = 32
tmpdir.join('a').write('z' * size)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 8
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = False
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._size = size
ase._encryption = None
ase2 = azmodels.StorageEntity('cont')
ase2._mode = azmodels.StorageModes.Block
ase2._name = 'name2'
ase2._size = size
ase2._encryption = None
ase.replica_targets = [ase2]
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud.hmac is None
assert ud.md5 is None
assert ud._outstanding_ops == 4 * 2
assert ud._completed_chunks is not None
assert ud._md5_cache is not None
assert ud._replica_counters is not None
assert ud.entity == ase
assert not ud.must_compute_md5
assert not ud.all_operations_completed
assert ud.last_block_num == -1
assert ud.is_resumable
assert not ud.remote_is_file
assert not ud.remote_is_page_blob
assert not ud.remote_is_append_blob
assert not ud.is_one_shot_block_blob
assert ud.requires_put_block_list
assert not ud.requires_non_encrypted_md5_put
assert not ud.requires_set_file_properties_md5
assert not ud.requires_access_tier_set
assert ud.requires_resize() == (False, ud._offset)
# test sym key
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._size = size
ase._encryption = mock.MagicMock()
opts.rsa_public_key = None
with pytest.raises(RuntimeError):
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
def test_cleanup_temporary_files(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 16
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd._allocate_disk_space()
dd.cleanup_all_temporary_files = mock.MagicMock()
dd.cleanup_all_temporary_files.side_effect = Exception
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.resume_file = pathlib.Path('abc')
d._dd_map[0] = dd
d._cleanup_temporary_files()
assert dd.final_path.exists()
lp = pathlib.Path(str(tmpdir.join('b')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 16
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd._allocate_disk_space()
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.resume_file = None
d._dd_map[0] = dd
d._cleanup_temporary_files()
assert not dd.final_path.exists()
lp = pathlib.Path(str(tmpdir.join('c')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 16
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd._allocate_disk_space()
dd.cleanup_all_temporary_files = mock.MagicMock()
dd.cleanup_all_temporary_files.side_effect = Exception
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.resume_file = None
d._dd_map[0] = dd
d._cleanup_temporary_files()
assert dd.final_path.exists()
def test_post_md5_skip_on_check(tmpdir):
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._download_total = 0
d._download_bytes_total = 0
d._md5_offload = mock.MagicMock()
lp = tmpdir.join('lpath').ensure(file=True)
lpath = str(lp)
rfile = azmodels.StorageEntity('cont')
rfile._md5 = 'abc'
rfile._client = mock.MagicMock()
rfile._client.primary_endpoint = 'ep'
rfile._name = 'name'
rfile._vio = None
rfile._size = 256
d._pre_md5_skip_on_check(lpath, rfile)
key = ops.Downloader.create_unique_transfer_operation_id(rfile)
d._transfer_set.add(key)
assert key in d._md5_map
d._post_md5_skip_on_check(key, lpath, None, True)
assert key not in d._md5_map
d._add_to_download_queue = mock.MagicMock()
d._pre_md5_skip_on_check(lpath, rfile)
d._transfer_set.add(key)
d._post_md5_skip_on_check(key, lpath, rfile._size, False)
assert d._add_to_download_queue.call_count == 1
def test_compute_total_chunks(tmpdir):
tmpdir.join('a').ensure(file=True)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 0
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES
with pytest.raises(RuntimeError):
ud._compute_total_chunks(1)
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES
ud._chunk_size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES
with pytest.raises(RuntimeError):
ud._compute_total_chunks(1)
ase._mode = azmodels.StorageModes.Append
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES
ud._chunk_size = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES
with pytest.raises(RuntimeError):
ud._compute_total_chunks(1)
def test_cleanup_all_temporary_files(tmpdir):
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 16
lp = pathlib.Path(str(tmpdir.join('a')))
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
data = b'0' * opts.chunk_size_bytes
d.write_unchecked_data(offsets, data)
assert len(d._unchecked_chunks) == 1
d.cleanup_all_temporary_files()
assert not d.final_path.exists()
assert not d._unchecked_chunks[0]['ucc'].file_path.exists()
lp = pathlib.Path(str(tmpdir.join('b')))
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
data = b'0' * opts.chunk_size_bytes
d.write_unchecked_hmac_data(offsets, data)
assert len(d._unchecked_chunks) == 1
d._unchecked_chunks[0]['ucc'].file_path.unlink()
d.cleanup_all_temporary_files()
assert not d.final_path.exists()
assert not d._unchecked_chunks[0]['ucc'].file_path.exists()
# go through except path
d.cleanup_all_temporary_files()
assert not d.final_path.exists()
def test_pre_md5_skip_on_check():
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._md5_offload = mock.MagicMock()
rfile = azmodels.StorageEntity('cont')
rfile._encryption = mock.MagicMock()
rfile._encryption.blobxfer_extensions = mock.MagicMock()
rfile._encryption.blobxfer_extensions.pre_encrypted_content_md5 = 'abc'
rfile._client = mock.MagicMock()
rfile._client.primary_endpoint = 'ep'
rfile._name = 'name'
rfile._size = 32
rfile._vio = mock.MagicMock()
rfile._vio.offset_start = 0
rfile._vio.total_size = 32
lpath = pathlib.Path('lpath')
key = ops.Downloader.create_unique_transfer_operation_id(rfile)
d._pre_md5_skip_on_check(lpath, rfile)
assert key in d._md5_map
rfile._name = 'name2'
rfile._vio = None
lpath = 'lpath2'
rfile._encryption = None
rfile._md5 = 'abc'
key = ops.Downloader.create_unique_transfer_operation_id(rfile)
d._pre_md5_skip_on_check(lpath, rfile)
assert key in d._md5_map
assert len(d._md5_map) == 2
def test_check_for_downloads_from_md5():
lpath = 'lpath'
rfile = azmodels.StorageEntity('cont')
rfile._md5 = 'abc'
rfile._client = mock.MagicMock()
rfile._client.primary_endpoint = 'ep'
rfile._name = 'name'
rfile._vio = None
rfile._size = 256
key = ops.Downloader.create_unique_transfer_operation_id(rfile)
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._md5_map[key] = rfile
d._transfer_set.add(key)
d._md5_offload = mock.MagicMock()
d._md5_offload.done_cv = multiprocessing.Condition()
d._md5_offload.pop_done_queue.side_effect = [
None,
(key, lpath, rfile._size, False),
]
d._add_to_download_queue = mock.MagicMock()
d._all_remote_files_processed = False
d._download_terminate = True
d._check_for_downloads_from_md5()
assert d._add_to_download_queue.call_count == 0
with mock.patch(
'blobxfer.operations.download.Downloader.'
'termination_check_md5',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(),
mock.MagicMock())
d._md5_map[key] = rfile
d._transfer_set.add(key)
d._md5_offload = mock.MagicMock()
d._md5_offload.done_cv = multiprocessing.Condition()
d._md5_offload.pop_done_queue.side_effect = [
None,
(key, lpath, rfile._size, False),
]
d._add_to_download_queue = mock.MagicMock()
patched_tc.side_effect = [False, False, True]
d._check_for_downloads_from_md5()
assert d._add_to_download_queue.call_count == 1
with mock.patch(
'blobxfer.operations.download.Downloader.'
'termination_check_md5',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(),
mock.MagicMock())
d._md5_map[key] = rfile
d._transfer_set.add(key)
d._md5_offload = mock.MagicMock()
d._md5_offload.done_cv = multiprocessing.Condition()
d._md5_offload.pop_done_queue.side_effect = [None]
d._add_to_download_queue = mock.MagicMock()
patched_tc.side_effect = [False, True, True]
d._check_for_downloads_from_md5()
assert d._add_to_download_queue.call_count == 0
def test_convert_vectored_io_slice_to_final_path_name():
lp = pathlib.Path('/local/path/abc.bxslice-0')
ase = azmodels.StorageEntity('cont')
ase._vio = mock.MagicMock()
ase._vio.slice_id = 0
fp = models.Descriptor.convert_vectored_io_slice_to_final_path_name(
lp, ase)
assert fp == pathlib.Path('/local/path/abc')
def test_downloaddescriptor(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 1024
ase._encryption = mock.MagicMock()
with pytest.raises(RuntimeError):
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
ase._encryption.symmetric_key = b'123'
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
assert not d._allocated
d._allocate_disk_space()
assert d.entity == ase
assert d.entity.is_encrypted
assert not d.must_compute_md5
assert d.hmac is not None
assert d._total_chunks == 64
assert d._offset == 0
assert d.final_path == lp
assert d._allocated
assert d.final_path.stat().st_size == ase._size - 16
d._allocate_disk_space()
assert d._allocated
d.final_path.unlink()
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
assert d._total_chunks == 2
assert d._allocated
assert d.final_path.stat().st_size == ase._size - 16
d.final_path.unlink()
ase._encryption = None
ase._size = 1024
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
assert d._allocated
assert d.final_path.stat().st_size == ase._size
# pre-existing file check
opts.chunk_size_bytes = 0
ase._size = 0
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
assert d._total_chunks == 0
assert d._allocated
assert d.final_path.stat().st_size == ase._size
def test_descriptor_compute_total_chunks():
opts = mock.MagicMock()
opts.dest_mode = azmodels.StorageModes.Auto
opts.mode = azmodels.StorageModes.Auto
src_ase = azmodels.StorageEntity('cont')
src_ase._mode = azmodels.StorageModes.Block
src_ase._name = 'name'
src_ase._size = 32
src_ase._encryption = None
dst_ase = azmodels.StorageEntity('cont2')
dst_ase._mode = azmodels.StorageModes.Block
dst_ase._name = 'name'
dst_ase._size = 32
dst_ase._encryption = None
dst_ase.replica_targets = [mock.MagicMock()]
d = synccopy.Descriptor(src_ase, dst_ase, None, opts, mock.MagicMock())
assert d._compute_total_chunks(0) == 1
def test_mark_unchecked_chunk_decrypted():
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 32
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(mock.MagicMock(), ase, opts, mock.MagicMock(), None)
d._unchecked_chunks[0] = {'decrypted': False}
d.mark_unchecked_chunk_decrypted(0)
assert d._unchecked_chunks[0]
def test_add_to_download_queue(tmpdir):
path = tmpdir.join('a')
lpath = pathlib.Path(str(path))
ase = azmodels.StorageEntity('cont')
ase._size = 1
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'abc'
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._spec.options.chunk_size_bytes = 1
d._add_to_download_queue(lpath, ase)
assert d._transfer_queue.qsize() == 1
assert path in d._dd_map
def test_downloaddescriptor_generate_view():
ase = azmodels.StorageEntity('cont')
ase._size = 1024
view, total_size = models.Descriptor.generate_view(ase)
assert view.fd_start == 0
assert view.fd_end == 1024
assert total_size == ase._size
ase._vio = mock.MagicMock()
ase._vio.offset_start = 2048
ase._vio.total_size = 3072
view, total_size = models.Descriptor.generate_view(ase)
assert view.fd_start == ase.vectored_io.offset_start
assert view.fd_end == ase.vectored_io.offset_start + ase._size
assert total_size == ase.vectored_io.total_size
def test_set_final_path_view():
lp = pathlib.Path('/local/path/abc.bxslice-0')
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 1024
ase._vio = mock.MagicMock()
ase._vio.slice_id = 0
ase._vio.total_size = 1024
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
total_size = d._set_final_path_view()
assert total_size == ase._size
def test_downloaddescriptor_allocate_disk_space_via_seek(tmpdir):
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 256
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), None)
with mock.patch('os.posix_fallocate') as patched_fallocate:
patched_fallocate.side_effect = [AttributeError()]
d._allocate_disk_space()
assert d._allocated
assert fp.exists()
assert fp.stat().st_size == ase._size
def test_operations(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._outstanding_ops = 1
d._unchecked_chunks = {0: None}
assert not d.all_operations_completed
d._outstanding_ops -= 1
d._unchecked_chunks.pop(0)
assert d.all_operations_completed
def test_write_data(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
data = b'0' * ase._size
d.write_data(offsets, data)
assert d.final_path.exists()
assert d.final_path.stat().st_size == len(data)
def test_update_resume_for_completed(tmpdir):
resumefile = pathlib.Path(str(tmpdir.join('resume')))
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._name = 'blob'
ase._client = mock.MagicMock()
rmgr = rops.DownloadResumeManager(resumefile)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
offsets, _ = d.next_offsets()
d._update_resume_for_completed()
dr = rmgr.get_record(ase)
assert dr.completed
def test_hmac_iv(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 256
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
ase._size = 128
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
iv = b'abc'
d.hmac_iv(iv)
assert d.hmac.update.call_count == 1
def test_descriptor_complete_offset_upload(tmpdir):
tmpdir.join('a').write('z' * 32)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 16
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._size = 32
ase._encryption = None
ase.replica_targets = [ase]
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._md5_cache[0] = 'md50'
ud._md5_cache[1] = 'md51'
ud.complete_offset_upload(0)
assert ud._outstanding_ops == 3
assert ud._replica_counters[0] == 0
ud.complete_offset_upload(1)
assert ud._outstanding_ops == 2
assert ud._replica_counters[1] == 0
# fill md5 cache with junk to trigger gc on next complete
for i in range(-30, -1):
ud._md5_cache[i] = ''
ud.complete_offset_upload(0)
assert ud._outstanding_ops == 1
assert 0 not in ud._replica_counters
assert len(ud._md5_cache) == 2
ud.complete_offset_upload(1)
assert ud._outstanding_ops == 0
assert 1 not in ud._replica_counters
assert len(ud._md5_cache) == 0
def test_descriptor_initialize_encryption(tmpdir):
tmpdir.join('a').write('z' * 32)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 16
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = 'abc'
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._size = 32
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud.hmac is not None
assert ud.entity.is_encrypted
def test_restore_file_attributes(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
lp.touch(mode=0o666, exist_ok=False)
lp.exists()
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._fileattr = mock.MagicMock()
ase._fileattr.mode = '0o100777'
ase._fileattr.uid = 1000
ase._fileattr.gid = 1000
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._restore_file_attributes()
stat = lp.stat()
assert str(oct(stat.st_mode)).replace('o', '') == \
ase._fileattr.mode.replace('o', '')
def test_write_unchecked_hmac_data(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 32
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
d.write_unchecked_hmac_data(offsets, b'0' * ase._size)
assert offsets.chunk_num in d._unchecked_chunks
ucc = d._unchecked_chunks[offsets.chunk_num]
assert ucc['ucc'].data_len == ase._size
assert ucc['ucc'].fd_start == offsets.fd_start
assert ucc['ucc'].file_path != d.final_path
assert ucc['ucc'].temp
assert not ucc['decrypted']
def test_process_download_descriptor_vio(tmpdir):
with mock.patch(
'blobxfer.models.download.Descriptor.all_operations_completed',
new_callable=mock.PropertyMock) as patched_aoc:
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(),
mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.File
ase._size = 16
ase._client = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase._name = 'name'
ase._vio = mock.MagicMock()
ase._vio.total_slices = 2
lp = pathlib.Path(str(tmpdir.join('b')))
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd.next_offsets = mock.MagicMock()
dd.next_offsets.return_value = (None, None)
patched_aoc.return_value = True
dd.finalize_file = mock.MagicMock()
key = ops.Downloader.create_unique_transfer_operation_id(ase)
d._transfer_set.add(key)
d._dd_map[str(lp)] = mock.MagicMock()
d._transfer_cc[dd.final_path] = mock.MagicMock()
d._process_download_descriptor(dd)
assert dd.finalize_file.call_count == 0
d._transfer_set.add(key)
d._dd_map[str(lp)] = mock.MagicMock()
d._transfer_cc[dd.final_path] = mock.MagicMock()
d._process_download_descriptor(dd)
assert dd.finalize_file.call_count == 1
def test_descriptor_hmac_data(tmpdir):
tmpdir.join('a').write('z' * 32)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 16
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = 'abc'
ase.replica_targets = [ase]
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud.hmac is not None
ud.hmac_data(b'\0')
def test_restore_file_lmt(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
lp.touch(mode=0o666, exist_ok=False)
lp.exists()
ts = util.datetime_now() - datetime.timedelta(seconds=60)
ts_posix = time.mktime(ts.timetuple())
stat = lp.stat()
assert stat.st_mtime != ts_posix
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
opts.restore_file_properties.lmt = True
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._lmt = ts
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._restore_file_lmt()
stat = lp.stat()
assert stat.st_mtime == ts_posix
def test_worker_thread_transfer(
patched_gbr, patched_gfr, patched_acdd, tmpdir):
# test disk set > max set length
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(
mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._process_download_descriptor = mock.MagicMock()
d._general_options.concurrency.disk_threads = 1
d._disk_set.add(0)
d._disk_set.add(1)
d._disk_set.add(2)
d._disk_set.add(3)
d._disk_set.add(4)
patched_tc.side_effect = [False, True]
d._worker_thread_transfer()
assert d._process_download_descriptor.call_count == 0
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._process_download_descriptor = mock.MagicMock()
d._download_terminate = True
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
d._worker_thread_transfer()
assert d._process_download_descriptor.call_count == 0
d._download_terminate = False
d._all_remote_files_processed = True
d._worker_thread_transfer()
assert d._process_download_descriptor.call_count == 0
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
patched_tc.side_effect = [False, False, True]
ase = azmodels.StorageEntity('cont')
ase._size = 16
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'abc'
lp = pathlib.Path(str(tmpdir.join('exc')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._transfer_queue = mock.MagicMock()
d._transfer_queue.get.side_effect = [queue.Empty, dd]
d._process_download_descriptor = mock.MagicMock()
d._process_download_descriptor.side_effect = RuntimeError('oops')
d._worker_thread_transfer()
assert len(d._exceptions) == 1
assert d._process_download_descriptor.call_count == 1
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
with mock.patch(
'blobxfer.models.download.Descriptor.'
'all_operations_completed',
new_callable=mock.PropertyMock) as patched_aoc:
d = ops.Downloader(
mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 16
ase._client = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase._name = 'name'
ase._vio = None
key = ops.Downloader.create_unique_transfer_operation_id(ase)
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'abc'
lp = pathlib.Path(str(tmpdir.join('a')))
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd.next_offsets = mock.MagicMock(
side_effect=[(None, 1), (None, 2)])
dd.finalize_integrity = mock.MagicMock()
dd.finalize_file = mock.MagicMock()
dd.perform_chunked_integrity_check = mock.MagicMock()
dd.all_operations_completed.side_effect = [False, True]
patched_aoc.side_effect = [False, True]
patched_tc.side_effect = [False, False, False, True]
d._dd_map[str(lp)] = dd
d._transfer_set.add(key)
d._transfer_queue = mock.MagicMock()
d._transfer_queue.get.side_effect = [queue.Empty, dd, dd]
d._worker_thread_transfer()
assert str(lp) not in d._dd_map
assert dd.finalize_file.call_count == 1
assert d._download_sofar == 1
assert d._download_bytes_sofar == 3
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(
mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.File
ase._size = 16
ase._client = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase._name = 'name'
ase._vio = None
key = ops.Downloader.create_unique_transfer_operation_id(ase)
patched_gfr.return_value = b'0' * ase._size
lp = pathlib.Path(str(tmpdir.join('b')))
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd.finalize_file = mock.MagicMock()
dd.perform_chunked_integrity_check = mock.MagicMock()
d._dd_map[str(lp)] = mock.MagicMock()
d._transfer_cc[dd.entity.path] = 0
d._transfer_set.add(key)
d._transfer_queue = mock.MagicMock()
d._transfer_queue.get.side_effect = [dd]
patched_tc.side_effect = [False, True]
d._spec.options.max_single_object_concurrency = 0
d._worker_thread_transfer()
assert len(d._disk_set) == 1
a, b, c = d._disk_queue.get()
d._process_data(a, b, c)
assert dd.perform_chunked_integrity_check.call_count == 1
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(
mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
d._spec.options.max_single_object_concurrency = 8
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Auto
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'abc'
ase._encryption.content_encryption_iv = b'0' * 16
ase._client = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase._name = 'name'
ase._vio = None
key = ops.Downloader.create_unique_transfer_operation_id(ase)
patched_gfr.return_value = b'0' * ase._size
lp = pathlib.Path(str(tmpdir.join('c')))
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd.finalize_file = mock.MagicMock()
dd.write_unchecked_hmac_data = mock.MagicMock()
dd.perform_chunked_integrity_check = mock.MagicMock()
d._crypto_offload = mock.MagicMock()
d._crypto_offload.add_decrypt_chunk = mock.MagicMock()
d._dd_map[str(lp)] = dd
d._transfer_cc[dd.entity.path] = 0
d._transfer_set.add(key)
d._transfer_queue = mock.MagicMock()
d._transfer_queue.get.side_effect = [dd]
patched_tc.side_effect = [False, True]
d._worker_thread_transfer()
assert len(d._disk_set) == 1
a, b, c = d._disk_queue.get()
d._process_data(a, b, c)
assert d._crypto_offload.add_decrypt_chunk.call_count == 1
assert dd.write_unchecked_hmac_data.call_count == 1
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(
mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.crypto_processes = 0
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
d._spec.options.max_single_object_concurrency = 8
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Auto
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'abc'
ase._encryption.content_encryption_iv = b'0' * 16
ase._client = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase._name = 'name'
ase._vio = None
key = ops.Downloader.create_unique_transfer_operation_id(ase)
patched_gfr.return_value = b'0' * ase._size
lp = pathlib.Path(str(tmpdir.join('d')))
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd.next_offsets()
dd.write_unchecked_hmac_data = mock.MagicMock()
dd.perform_chunked_integrity_check = mock.MagicMock()
dd.mark_unchecked_chunk_decrypted = mock.MagicMock()
patched_acdd.return_value = b'0' * 16
d._dd_map[str(lp)] = mock.MagicMock()
d._transfer_cc[dd.entity.path] = 0
d._transfer_set.add(key)
d._transfer_queue = mock.MagicMock()
d._transfer_queue.get.side_effect = [dd, dd]
patched_tc.side_effect = [False, True]
d._worker_thread_transfer()
assert len(d._disk_set) == 1
a, b, c = d._disk_queue.get()
d._process_data(a, b, c)
assert patched_acdd.call_count == 1
assert dd.write_unchecked_hmac_data.call_count == 1
assert dd.perform_chunked_integrity_check.call_count == 1
