def test_descriptor_complete_offset_upload(): opts = mock.MagicMock() opts.dest_mode = azmodels.StorageModes.Auto opts.mode = azmodels.StorageModes.Auto src_ase = azmodels.StorageEntity('cont') src_ase._mode = azmodels.StorageModes.Block src_ase._name = 'name' src_ase._size = 32 src_ase._encryption = None dst_ase = azmodels.StorageEntity('cont2') dst_ase._mode = azmodels.StorageModes.Block dst_ase._name = 'name' dst_ase._size = 32 dst_ase._encryption = None dst_ase.replica_targets = [mock.MagicMock()] d = synccopy.Descriptor(src_ase, dst_ase, None, opts, mock.MagicMock()) d.complete_offset_upload(0) assert d._outstanding_ops == 1 d.complete_offset_upload(0) assert d._outstanding_ops == 0 assert 0 not in d._replica_counters
def test_descriptor_compute_chunk_size(): opts = mock.MagicMock() opts.dest_mode = azmodels.StorageModes.Auto opts.mode = azmodels.StorageModes.Auto src_ase = azmodels.StorageEntity('cont') src_ase._mode = azmodels.StorageModes.Block src_ase._name = 'name' src_ase._size = 32 src_ase._encryption = None dst_ase = azmodels.StorageEntity('cont2') dst_ase._mode = azmodels.StorageModes.Block dst_ase._name = 'name' dst_ase._size = 32 dst_ase._encryption = None dst_ase.replica_targets = [mock.MagicMock()] d = synccopy.Descriptor(src_ase, dst_ase, None, opts, mock.MagicMock()) assert d._compute_chunk_size() == \ synccopy._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES d = synccopy.Descriptor(src_ase, dst_ase, [], opts, mock.MagicMock()) assert d._compute_chunk_size() == d.src_entity.size b = mock.MagicMock() b.size = 1 d = synccopy.Descriptor(src_ase, dst_ase, [b], opts, mock.MagicMock()) assert d._compute_chunk_size() == 1 d = synccopy.Descriptor(src_ase, dst_ase, [b, b], opts, mock.MagicMock()) assert d._compute_chunk_size() == -1
def test_descriptor(): opts = mock.MagicMock() opts.dest_mode = azmodels.StorageModes.Auto opts.mode = azmodels.StorageModes.Auto src_ase = azmodels.StorageEntity('cont') src_ase._mode = azmodels.StorageModes.Block src_ase._name = 'name' src_ase._size = 32 src_ase._encryption = None dst_ase = azmodels.StorageEntity('cont2') dst_ase._mode = azmodels.StorageModes.Block dst_ase._name = 'name' dst_ase._size = 32 dst_ase._encryption = None dst_ase.replica_targets = [mock.MagicMock()] d = synccopy.Descriptor(src_ase, dst_ase, None, opts, mock.MagicMock()) assert d._offset == 0 assert d._chunk_num == 0 assert not d._finalized assert d._src_block_list is None assert d.src_entity == src_ase assert d.dst_entity == dst_ase assert not d.all_operations_completed assert d.is_resumable assert d.last_block_num == -1 assert not d.remote_is_file assert not d.remote_is_page_blob assert not d.remote_is_append_blob assert d.is_one_shot_block_blob assert not d.requires_put_block_list
def test_descriptor(tmpdir): size = 32 tmpdir.join('a').write('z' * size) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 8 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = False opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._size = size ase._encryption = None ase2 = azmodels.StorageEntity('cont') ase2._mode = azmodels.StorageModes.Block ase2._name = 'name2' ase2._size = size ase2._encryption = None ase.replica_targets = [ase2] ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud.hmac is None assert ud.md5 is None assert ud._outstanding_ops == 4 * 2 assert ud._completed_chunks is not None assert ud._md5_cache is not None assert ud._replica_counters is not None assert ud.entity == ase assert not ud.must_compute_md5 assert not ud.all_operations_completed assert ud.last_block_num == -1 assert ud.is_resumable assert not ud.remote_is_file assert not ud.remote_is_page_blob assert not ud.remote_is_append_blob assert not ud.is_one_shot_block_blob assert ud.requires_put_block_list assert not ud.requires_non_encrypted_md5_put assert not ud.requires_set_file_properties_md5 assert not ud.requires_access_tier_set assert ud.requires_resize() == (False, ud._offset) # test sym key ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._size = size ase._encryption = mock.MagicMock() opts.rsa_public_key = None with pytest.raises(RuntimeError): ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock())
def test_cleanup_temporary_files(tmpdir): lp = pathlib.Path(str(tmpdir.join('a'))) opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 16 dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) dd._allocate_disk_space() dd.cleanup_all_temporary_files = mock.MagicMock() dd.cleanup_all_temporary_files.side_effect = Exception d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._general_options.resume_file = pathlib.Path('abc') d._dd_map[0] = dd d._cleanup_temporary_files() assert dd.final_path.exists() lp = pathlib.Path(str(tmpdir.join('b'))) opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 16 dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) dd._allocate_disk_space() d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._general_options.resume_file = None d._dd_map[0] = dd d._cleanup_temporary_files() assert not dd.final_path.exists() lp = pathlib.Path(str(tmpdir.join('c'))) opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 16 dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) dd._allocate_disk_space() dd.cleanup_all_temporary_files = mock.MagicMock() dd.cleanup_all_temporary_files.side_effect = Exception d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._general_options.resume_file = None d._dd_map[0] = dd d._cleanup_temporary_files() assert dd.final_path.exists()
def test_post_md5_skip_on_check(tmpdir): d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._download_total = 0 d._download_bytes_total = 0 d._md5_offload = mock.MagicMock() lp = tmpdir.join('lpath').ensure(file=True) lpath = str(lp) rfile = azmodels.StorageEntity('cont') rfile._md5 = 'abc' rfile._client = mock.MagicMock() rfile._client.primary_endpoint = 'ep' rfile._name = 'name' rfile._vio = None rfile._size = 256 d._pre_md5_skip_on_check(lpath, rfile) key = ops.Downloader.create_unique_transfer_operation_id(rfile) d._transfer_set.add(key) assert key in d._md5_map d._post_md5_skip_on_check(key, lpath, None, True) assert key not in d._md5_map d._add_to_download_queue = mock.MagicMock() d._pre_md5_skip_on_check(lpath, rfile) d._transfer_set.add(key) d._post_md5_skip_on_check(key, lpath, rfile._size, False) assert d._add_to_download_queue.call_count == 1
def test_compute_total_chunks(tmpdir): tmpdir.join('a').ensure(file=True) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 0 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES with pytest.raises(RuntimeError): ud._compute_total_chunks(1) ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES ud._chunk_size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES with pytest.raises(RuntimeError): ud._compute_total_chunks(1) ase._mode = azmodels.StorageModes.Append ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES ud._chunk_size = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES with pytest.raises(RuntimeError): ud._compute_total_chunks(1)
def test_cleanup_all_temporary_files(tmpdir): opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 16 lp = pathlib.Path(str(tmpdir.join('a'))) d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) offsets, _ = d.next_offsets() data = b'0' * opts.chunk_size_bytes d.write_unchecked_data(offsets, data) assert len(d._unchecked_chunks) == 1 d.cleanup_all_temporary_files() assert not d.final_path.exists() assert not d._unchecked_chunks[0]['ucc'].file_path.exists() lp = pathlib.Path(str(tmpdir.join('b'))) d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) offsets, _ = d.next_offsets() data = b'0' * opts.chunk_size_bytes d.write_unchecked_hmac_data(offsets, data) assert len(d._unchecked_chunks) == 1 d._unchecked_chunks[0]['ucc'].file_path.unlink() d.cleanup_all_temporary_files() assert not d.final_path.exists() assert not d._unchecked_chunks[0]['ucc'].file_path.exists() # go through except path d.cleanup_all_temporary_files() assert not d.final_path.exists()
def test_pre_md5_skip_on_check(): d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._md5_offload = mock.MagicMock() rfile = azmodels.StorageEntity('cont') rfile._encryption = mock.MagicMock() rfile._encryption.blobxfer_extensions = mock.MagicMock() rfile._encryption.blobxfer_extensions.pre_encrypted_content_md5 = 'abc' rfile._client = mock.MagicMock() rfile._client.primary_endpoint = 'ep' rfile._name = 'name' rfile._size = 32 rfile._vio = mock.MagicMock() rfile._vio.offset_start = 0 rfile._vio.total_size = 32 lpath = pathlib.Path('lpath') key = ops.Downloader.create_unique_transfer_operation_id(rfile) d._pre_md5_skip_on_check(lpath, rfile) assert key in d._md5_map rfile._name = 'name2' rfile._vio = None lpath = 'lpath2' rfile._encryption = None rfile._md5 = 'abc' key = ops.Downloader.create_unique_transfer_operation_id(rfile) d._pre_md5_skip_on_check(lpath, rfile) assert key in d._md5_map assert len(d._md5_map) == 2
def test_check_for_downloads_from_md5(): lpath = 'lpath' rfile = azmodels.StorageEntity('cont') rfile._md5 = 'abc' rfile._client = mock.MagicMock() rfile._client.primary_endpoint = 'ep' rfile._name = 'name' rfile._vio = None rfile._size = 256 key = ops.Downloader.create_unique_transfer_operation_id(rfile) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._md5_map[key] = rfile d._transfer_set.add(key) d._md5_offload = mock.MagicMock() d._md5_offload.done_cv = multiprocessing.Condition() d._md5_offload.pop_done_queue.side_effect = [ None, (key, lpath, rfile._size, False), ] d._add_to_download_queue = mock.MagicMock() d._all_remote_files_processed = False d._download_terminate = True d._check_for_downloads_from_md5() assert d._add_to_download_queue.call_count == 0 with mock.patch( 'blobxfer.operations.download.Downloader.' 'termination_check_md5', new_callable=mock.PropertyMock) as patched_tc: d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._md5_map[key] = rfile d._transfer_set.add(key) d._md5_offload = mock.MagicMock() d._md5_offload.done_cv = multiprocessing.Condition() d._md5_offload.pop_done_queue.side_effect = [ None, (key, lpath, rfile._size, False), ] d._add_to_download_queue = mock.MagicMock() patched_tc.side_effect = [False, False, True] d._check_for_downloads_from_md5() assert d._add_to_download_queue.call_count == 1 with mock.patch( 'blobxfer.operations.download.Downloader.' 'termination_check_md5', new_callable=mock.PropertyMock) as patched_tc: d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._md5_map[key] = rfile d._transfer_set.add(key) d._md5_offload = mock.MagicMock() d._md5_offload.done_cv = multiprocessing.Condition() d._md5_offload.pop_done_queue.side_effect = [None] d._add_to_download_queue = mock.MagicMock() patched_tc.side_effect = [False, True, True] d._check_for_downloads_from_md5() assert d._add_to_download_queue.call_count == 0
def test_convert_vectored_io_slice_to_final_path_name(): lp = pathlib.Path('/local/path/abc.bxslice-0') ase = azmodels.StorageEntity('cont') ase._vio = mock.MagicMock() ase._vio.slice_id = 0 fp = models.Descriptor.convert_vectored_io_slice_to_final_path_name( lp, ase) assert fp == pathlib.Path('/local/path/abc')
def test_downloaddescriptor(tmpdir): lp = pathlib.Path(str(tmpdir.join('a'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 1024 ase._encryption = mock.MagicMock() with pytest.raises(RuntimeError): d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) ase._encryption.symmetric_key = b'123' d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) assert not d._allocated d._allocate_disk_space() assert d.entity == ase assert d.entity.is_encrypted assert not d.must_compute_md5 assert d.hmac is not None assert d._total_chunks == 64 assert d._offset == 0 assert d.final_path == lp assert d._allocated assert d.final_path.stat().st_size == ase._size - 16 d._allocate_disk_space() assert d._allocated d.final_path.unlink() ase._size = 32 d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._allocate_disk_space() assert d._total_chunks == 2 assert d._allocated assert d.final_path.stat().st_size == ase._size - 16 d.final_path.unlink() ase._encryption = None ase._size = 1024 d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._allocate_disk_space() assert d._allocated assert d.final_path.stat().st_size == ase._size # pre-existing file check opts.chunk_size_bytes = 0 ase._size = 0 d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._allocate_disk_space() assert d._total_chunks == 0 assert d._allocated assert d.final_path.stat().st_size == ase._size
def test_descriptor_compute_total_chunks(): opts = mock.MagicMock() opts.dest_mode = azmodels.StorageModes.Auto opts.mode = azmodels.StorageModes.Auto src_ase = azmodels.StorageEntity('cont') src_ase._mode = azmodels.StorageModes.Block src_ase._name = 'name' src_ase._size = 32 src_ase._encryption = None dst_ase = azmodels.StorageEntity('cont2') dst_ase._mode = azmodels.StorageModes.Block dst_ase._name = 'name' dst_ase._size = 32 dst_ase._encryption = None dst_ase.replica_targets = [mock.MagicMock()] d = synccopy.Descriptor(src_ase, dst_ase, None, opts, mock.MagicMock()) assert d._compute_total_chunks(0) == 1
def test_mark_unchecked_chunk_decrypted(): opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 32 ase = azmodels.StorageEntity('cont') ase._size = 32 d = models.Descriptor(mock.MagicMock(), ase, opts, mock.MagicMock(), None) d._unchecked_chunks[0] = {'decrypted': False} d.mark_unchecked_chunk_decrypted(0) assert d._unchecked_chunks[0]
def test_add_to_download_queue(tmpdir): path = tmpdir.join('a') lpath = pathlib.Path(str(path)) ase = azmodels.StorageEntity('cont') ase._size = 1 ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = b'abc' d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._spec.options.chunk_size_bytes = 1 d._add_to_download_queue(lpath, ase) assert d._transfer_queue.qsize() == 1 assert path in d._dd_map
def test_downloaddescriptor_generate_view(): ase = azmodels.StorageEntity('cont') ase._size = 1024 view, total_size = models.Descriptor.generate_view(ase) assert view.fd_start == 0 assert view.fd_end == 1024 assert total_size == ase._size ase._vio = mock.MagicMock() ase._vio.offset_start = 2048 ase._vio.total_size = 3072 view, total_size = models.Descriptor.generate_view(ase) assert view.fd_start == ase.vectored_io.offset_start assert view.fd_end == ase.vectored_io.offset_start + ase._size assert total_size == ase.vectored_io.total_size
def test_set_final_path_view(): lp = pathlib.Path('/local/path/abc.bxslice-0') opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 1024 ase._vio = mock.MagicMock() ase._vio.slice_id = 0 ase._vio.total_size = 1024 d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) total_size = d._set_final_path_view() assert total_size == ase._size
def test_downloaddescriptor_allocate_disk_space_via_seek(tmpdir): fp = pathlib.Path(str(tmpdir.join('fp'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 256 ase = azmodels.StorageEntity('cont') ase._size = 128 ase._name = 'blob' d = models.Descriptor(fp, ase, opts, mock.MagicMock(), None) with mock.patch('os.posix_fallocate') as patched_fallocate: patched_fallocate.side_effect = [AttributeError()] d._allocate_disk_space() assert d._allocated assert fp.exists() assert fp.stat().st_size == ase._size
def test_operations(tmpdir): lp = pathlib.Path(str(tmpdir.join('a'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._outstanding_ops = 1 d._unchecked_chunks = {0: None} assert not d.all_operations_completed d._outstanding_ops -= 1 d._unchecked_chunks.pop(0) assert d.all_operations_completed
def test_write_data(tmpdir): lp = pathlib.Path(str(tmpdir.join('a'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) offsets, _ = d.next_offsets() data = b'0' * ase._size d.write_data(offsets, data) assert d.final_path.exists() assert d.final_path.stat().st_size == len(data)
def test_update_resume_for_completed(tmpdir): resumefile = pathlib.Path(str(tmpdir.join('resume'))) fp = pathlib.Path(str(tmpdir.join('fp'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 ase._name = 'blob' ase._client = mock.MagicMock() rmgr = rops.DownloadResumeManager(resumefile) d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) offsets, _ = d.next_offsets() d._update_resume_for_completed() dr = rmgr.get_record(ase) assert dr.completed
def test_hmac_iv(tmpdir): lp = pathlib.Path(str(tmpdir.join('a'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 256 ase = azmodels.StorageEntity('cont') ase._size = 128 ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = b'123' ase._size = 128 d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) iv = b'abc' d.hmac_iv(iv) assert d.hmac.update.call_count == 1
def test_descriptor_complete_offset_upload(tmpdir): tmpdir.join('a').write('z' * 32) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 16 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._size = 32 ase._encryption = None ase.replica_targets = [ase] ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._md5_cache[0] = 'md50' ud._md5_cache[1] = 'md51' ud.complete_offset_upload(0) assert ud._outstanding_ops == 3 assert ud._replica_counters[0] == 0 ud.complete_offset_upload(1) assert ud._outstanding_ops == 2 assert ud._replica_counters[1] == 0 # fill md5 cache with junk to trigger gc on next complete for i in range(-30, -1): ud._md5_cache[i] = '' ud.complete_offset_upload(0) assert ud._outstanding_ops == 1 assert 0 not in ud._replica_counters assert len(ud._md5_cache) == 2 ud.complete_offset_upload(1) assert ud._outstanding_ops == 0 assert 1 not in ud._replica_counters assert len(ud._md5_cache) == 0
def test_descriptor_initialize_encryption(tmpdir): tmpdir.join('a').write('z' * 32) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 16 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = 'abc' ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._size = 32 ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud.hmac is not None assert ud.entity.is_encrypted
def test_restore_file_attributes(tmpdir): lp = pathlib.Path(str(tmpdir.join('a'))) lp.touch(mode=0o666, exist_ok=False) lp.exists() opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 ase._fileattr = mock.MagicMock() ase._fileattr.mode = '0o100777' ase._fileattr.uid = 1000 ase._fileattr.gid = 1000 d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._restore_file_attributes() stat = lp.stat() assert str(oct(stat.st_mode)).replace('o', '') == \ ase._fileattr.mode.replace('o', '')
def test_write_unchecked_hmac_data(tmpdir): lp = pathlib.Path(str(tmpdir.join('a'))) opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 32 ase = azmodels.StorageEntity('cont') ase._size = 32 d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) offsets, _ = d.next_offsets() d.write_unchecked_hmac_data(offsets, b'0' * ase._size) assert offsets.chunk_num in d._unchecked_chunks ucc = d._unchecked_chunks[offsets.chunk_num] assert ucc['ucc'].data_len == ase._size assert ucc['ucc'].fd_start == offsets.fd_start assert ucc['ucc'].file_path != d.final_path assert ucc['ucc'].temp assert not ucc['decrypted']
def test_process_download_descriptor_vio(tmpdir): with mock.patch( 'blobxfer.models.download.Descriptor.all_operations_completed', new_callable=mock.PropertyMock) as patched_aoc: d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._general_options.concurrency.transfer_threads = 1 d._general_options.concurrency.disk_threads = 1 opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.File ase._size = 16 ase._client = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase._name = 'name' ase._vio = mock.MagicMock() ase._vio.total_slices = 2 lp = pathlib.Path(str(tmpdir.join('b'))) dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) dd.next_offsets = mock.MagicMock() dd.next_offsets.return_value = (None, None) patched_aoc.return_value = True dd.finalize_file = mock.MagicMock() key = ops.Downloader.create_unique_transfer_operation_id(ase) d._transfer_set.add(key) d._dd_map[str(lp)] = mock.MagicMock() d._transfer_cc[dd.final_path] = mock.MagicMock() d._process_download_descriptor(dd) assert dd.finalize_file.call_count == 0 d._transfer_set.add(key) d._dd_map[str(lp)] = mock.MagicMock() d._transfer_cc[dd.final_path] = mock.MagicMock() d._process_download_descriptor(dd) assert dd.finalize_file.call_count == 1
def test_descriptor_hmac_data(tmpdir): tmpdir.join('a').write('z' * 32) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 16 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._size = 32 ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = 'abc' ase.replica_targets = [ase] ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud.hmac is not None ud.hmac_data(b'\0')
def test_restore_file_lmt(tmpdir): lp = pathlib.Path(str(tmpdir.join('a'))) lp.touch(mode=0o666, exist_ok=False) lp.exists() ts = util.datetime_now() - datetime.timedelta(seconds=60) ts_posix = time.mktime(ts.timetuple()) stat = lp.stat() assert stat.st_mtime != ts_posix opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 opts.restore_file_properties.lmt = True ase = azmodels.StorageEntity('cont') ase._size = 32 ase._lmt = ts d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._restore_file_lmt() stat = lp.stat() assert stat.st_mtime == ts_posix
def test_worker_thread_transfer( patched_gbr, patched_gfr, patched_acdd, tmpdir): # test disk set > max set length with mock.patch( 'blobxfer.operations.download.Downloader.termination_check', new_callable=mock.PropertyMock) as patched_tc: d = ops.Downloader( mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._process_download_descriptor = mock.MagicMock() d._general_options.concurrency.disk_threads = 1 d._disk_set.add(0) d._disk_set.add(1) d._disk_set.add(2) d._disk_set.add(3) d._disk_set.add(4) patched_tc.side_effect = [False, True] d._worker_thread_transfer() assert d._process_download_descriptor.call_count == 0 d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._process_download_descriptor = mock.MagicMock() d._download_terminate = True d._general_options.concurrency.transfer_threads = 1 d._general_options.concurrency.disk_threads = 1 d._worker_thread_transfer() assert d._process_download_descriptor.call_count == 0 d._download_terminate = False d._all_remote_files_processed = True d._worker_thread_transfer() assert d._process_download_descriptor.call_count == 0 with mock.patch( 'blobxfer.operations.download.Downloader.termination_check', new_callable=mock.PropertyMock) as patched_tc: patched_tc.side_effect = [False, False, True] ase = azmodels.StorageEntity('cont') ase._size = 16 ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = b'abc' lp = pathlib.Path(str(tmpdir.join('exc'))) opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._transfer_queue = mock.MagicMock() d._transfer_queue.get.side_effect = [queue.Empty, dd] d._process_download_descriptor = mock.MagicMock() d._process_download_descriptor.side_effect = RuntimeError('oops') d._worker_thread_transfer() assert len(d._exceptions) == 1 assert d._process_download_descriptor.call_count == 1 with mock.patch( 'blobxfer.operations.download.Downloader.termination_check', new_callable=mock.PropertyMock) as patched_tc: with mock.patch( 'blobxfer.models.download.Descriptor.' 'all_operations_completed', new_callable=mock.PropertyMock) as patched_aoc: d = ops.Downloader( mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._general_options.concurrency.transfer_threads = 1 d._general_options.concurrency.disk_threads = 1 opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 16 ase._client = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase._name = 'name' ase._vio = None key = ops.Downloader.create_unique_transfer_operation_id(ase) ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = b'abc' lp = pathlib.Path(str(tmpdir.join('a'))) dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) dd.next_offsets = mock.MagicMock( side_effect=[(None, 1), (None, 2)]) dd.finalize_integrity = mock.MagicMock() dd.finalize_file = mock.MagicMock() dd.perform_chunked_integrity_check = mock.MagicMock() dd.all_operations_completed.side_effect = [False, True] patched_aoc.side_effect = [False, True] patched_tc.side_effect = [False, False, False, True] d._dd_map[str(lp)] = dd d._transfer_set.add(key) d._transfer_queue = mock.MagicMock() d._transfer_queue.get.side_effect = [queue.Empty, dd, dd] d._worker_thread_transfer() assert str(lp) not in d._dd_map assert dd.finalize_file.call_count == 1 assert d._download_sofar == 1 assert d._download_bytes_sofar == 3 with mock.patch( 'blobxfer.operations.download.Downloader.termination_check', new_callable=mock.PropertyMock) as patched_tc: d = ops.Downloader( mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._general_options.concurrency.transfer_threads = 1 d._general_options.concurrency.disk_threads = 1 opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.File ase._size = 16 ase._client = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase._name = 'name' ase._vio = None key = ops.Downloader.create_unique_transfer_operation_id(ase) patched_gfr.return_value = b'0' * ase._size lp = pathlib.Path(str(tmpdir.join('b'))) dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) dd.finalize_file = mock.MagicMock() dd.perform_chunked_integrity_check = mock.MagicMock() d._dd_map[str(lp)] = mock.MagicMock() d._transfer_cc[dd.entity.path] = 0 d._transfer_set.add(key) d._transfer_queue = mock.MagicMock() d._transfer_queue.get.side_effect = [dd] patched_tc.side_effect = [False, True] d._spec.options.max_single_object_concurrency = 0 d._worker_thread_transfer() assert len(d._disk_set) == 1 a, b, c = d._disk_queue.get() d._process_data(a, b, c) assert dd.perform_chunked_integrity_check.call_count == 1 with mock.patch( 'blobxfer.operations.download.Downloader.termination_check', new_callable=mock.PropertyMock) as patched_tc: d = ops.Downloader( mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._general_options.concurrency.transfer_threads = 1 d._general_options.concurrency.disk_threads = 1 d._spec.options.max_single_object_concurrency = 8 opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Auto ase._size = 32 ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = b'abc' ase._encryption.content_encryption_iv = b'0' * 16 ase._client = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase._name = 'name' ase._vio = None key = ops.Downloader.create_unique_transfer_operation_id(ase) patched_gfr.return_value = b'0' * ase._size lp = pathlib.Path(str(tmpdir.join('c'))) dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) dd.finalize_file = mock.MagicMock() dd.write_unchecked_hmac_data = mock.MagicMock() dd.perform_chunked_integrity_check = mock.MagicMock() d._crypto_offload = mock.MagicMock() d._crypto_offload.add_decrypt_chunk = mock.MagicMock() d._dd_map[str(lp)] = dd d._transfer_cc[dd.entity.path] = 0 d._transfer_set.add(key) d._transfer_queue = mock.MagicMock() d._transfer_queue.get.side_effect = [dd] patched_tc.side_effect = [False, True] d._worker_thread_transfer() assert len(d._disk_set) == 1 a, b, c = d._disk_queue.get() d._process_data(a, b, c) assert d._crypto_offload.add_decrypt_chunk.call_count == 1 assert dd.write_unchecked_hmac_data.call_count == 1 with mock.patch( 'blobxfer.operations.download.Downloader.termination_check', new_callable=mock.PropertyMock) as patched_tc: d = ops.Downloader( mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._general_options.concurrency.crypto_processes = 0 d._general_options.concurrency.transfer_threads = 1 d._general_options.concurrency.disk_threads = 1 d._spec.options.max_single_object_concurrency = 8 opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Auto ase._size = 32 ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = b'abc' ase._encryption.content_encryption_iv = b'0' * 16 ase._client = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase._name = 'name' ase._vio = None key = ops.Downloader.create_unique_transfer_operation_id(ase) patched_gfr.return_value = b'0' * ase._size lp = pathlib.Path(str(tmpdir.join('d'))) dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) dd.next_offsets() dd.write_unchecked_hmac_data = mock.MagicMock() dd.perform_chunked_integrity_check = mock.MagicMock() dd.mark_unchecked_chunk_decrypted = mock.MagicMock() patched_acdd.return_value = b'0' * 16 d._dd_map[str(lp)] = mock.MagicMock() d._transfer_cc[dd.entity.path] = 0 d._transfer_set.add(key) d._transfer_queue = mock.MagicMock() d._transfer_queue.get.side_effect = [dd, dd] patched_tc.side_effect = [False, True] d._worker_thread_transfer() assert len(d._disk_set) == 1 a, b, c = d._disk_queue.get() d._process_data(a, b, c) assert patched_acdd.call_count == 1 assert dd.write_unchecked_hmac_data.call_count == 1 assert dd.perform_chunked_integrity_check.call_count == 1