def test_compute_total_chunks(tmpdir): tmpdir.join('a').ensure(file=True) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 0 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES with pytest.raises(RuntimeError): ud._compute_total_chunks(1) ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES ud._chunk_size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES with pytest.raises(RuntimeError): ud._compute_total_chunks(1) ase._mode = azmodels.StorageModes.Append ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES ud._chunk_size = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES with pytest.raises(RuntimeError): ud._compute_total_chunks(1)
def test_descriptor(tmpdir): size = 32 tmpdir.join('a').write('z' * size) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 8 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = False opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._size = size ase._encryption = None ase2 = azmodels.StorageEntity('cont') ase2._mode = azmodels.StorageModes.Block ase2._name = 'name2' ase2._size = size ase2._encryption = None ase.replica_targets = [ase2] ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud.hmac is None assert ud.md5 is None assert ud._outstanding_ops == 4 * 2 assert ud._completed_chunks is not None assert ud._md5_cache is not None assert ud._replica_counters is not None assert ud.entity == ase assert not ud.must_compute_md5 assert not ud.all_operations_completed assert ud.last_block_num == -1 assert ud.is_resumable assert not ud.remote_is_file assert not ud.remote_is_page_blob assert not ud.remote_is_append_blob assert not ud.is_one_shot_block_blob assert ud.requires_put_block_list assert not ud.requires_non_encrypted_md5_put assert not ud.requires_set_file_properties_md5 assert not ud.requires_access_tier_set assert ud.requires_resize() == (False, ud._offset) # test sym key ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._size = size ase._encryption = mock.MagicMock() opts.rsa_public_key = None with pytest.raises(RuntimeError): ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock())
def test_localpath(tmpdir): tmpdir.join('a').write('zz') pp = pathlib.Path(str(tmpdir)) rp = pathlib.Path('a') file = pp / rp stat = file.stat() lp = upload.LocalPath(pp, rp, use_stdin=True, view=None) assert lp.absolute_path == file assert lp.size == 0 assert lp.total_size == 0 assert lp.lmt == 0 assert lp.mode.replace('o', '') == '00' assert lp.uid == 0 assert lp.gid == 0 lp = upload.LocalPath(pp, rp, use_stdin=False, view=None) assert lp.absolute_path == file assert lp.size == stat.st_size assert lp.total_size == stat.st_size assert lp.lmt == stat.st_mtime assert lp.mode.replace('o', '') == str(oct(stat.st_mode)).replace('o', '') assert lp.uid == stat.st_uid assert lp.gid == stat.st_gid lpview = upload.LocalPathView( fd_start=1, fd_end=2, slice_num=1, mode=upload.VectoredIoDistributionMode.Stripe, total_slices=2, next=None, ) lp = upload.LocalPath(pp, rp, use_stdin=False, view=lpview) assert lp.absolute_path == file assert lp.size == 1 assert lp.total_size == stat.st_size assert lp.lmt == stat.st_mtime assert lp.mode.replace('o', '') == str(oct(stat.st_mode)).replace('o', '') assert lp.uid == stat.st_uid assert lp.gid == stat.st_gid
def test_descriptor_initialize_encryption(tmpdir): tmpdir.join('a').write('z' * 32) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 16 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = 'abc' ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._size = 32 ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud.hmac is not None assert ud.entity.is_encrypted
def test_descriptor_complete_offset_upload(tmpdir): tmpdir.join('a').write('z' * 32) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 16 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._size = 32 ase._encryption = None ase.replica_targets = [ase] ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._md5_cache[0] = 'md50' ud._md5_cache[1] = 'md51' ud.complete_offset_upload(0) assert ud._outstanding_ops == 3 assert ud._replica_counters[0] == 0 ud.complete_offset_upload(1) assert ud._outstanding_ops == 2 assert ud._replica_counters[1] == 0 # fill md5 cache with junk to trigger gc on next complete for i in range(-30, -1): ud._md5_cache[i] = '' ud.complete_offset_upload(0) assert ud._outstanding_ops == 1 assert 0 not in ud._replica_counters assert len(ud._md5_cache) == 2 ud.complete_offset_upload(1) assert ud._outstanding_ops == 0 assert 1 not in ud._replica_counters assert len(ud._md5_cache) == 0
def test_descriptor_hmac_data(tmpdir): tmpdir.join('a').write('z' * 32) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 16 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._size = 32 ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = 'abc' ase.replica_targets = [ase] ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud.hmac is not None ud.hmac_data(b'\0')
def test_descriptor_next_offsets(tmpdir): tmpdir.join('a').write('ab') lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 1 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None # test normal ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._resume = mock.MagicMock() ud._resume.return_value = None offsets, rb = ud.next_offsets() assert rb is None assert offsets.chunk_num == 0 assert offsets.num_bytes == 1 assert offsets.range_start == 0 assert offsets.range_end == 0 assert not offsets.pad assert ud._offset == 1 assert ud._chunk_num == 1 offsets, rb = ud.next_offsets() assert rb is None assert offsets.chunk_num == 1 assert offsets.num_bytes == 1 assert offsets.range_start == 1 assert offsets.range_end == 1 assert not offsets.pad assert ud._offset == 2 assert ud._chunk_num == 2 offsets, rb = ud.next_offsets() assert rb is None assert offsets is None # test chunk size exceeds size lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts.chunk_size_bytes = 3 ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._chunk_size = 3 ud._resume = mock.MagicMock() ud._resume.return_value = None offsets, rb = ud.next_offsets() assert rb is None assert offsets.chunk_num == 0 assert offsets.num_bytes == 2 assert offsets.range_start == 0 assert offsets.range_end == 1 assert not offsets.pad assert ud._offset == 2 assert ud._chunk_num == 1 # test encrypted tmpdir.join('a').write('z' * 16) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts.chunk_size_bytes = 16 opts.rsa_public_key = 'abc' ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._resume = mock.MagicMock() ud._resume.return_value = None offsets, rb = ud.next_offsets() assert rb is None assert offsets.chunk_num == 0 assert offsets.num_bytes == 16 assert offsets.range_start == 0 assert offsets.range_end == 15 assert not offsets.pad assert ud._offset == 16 assert ud._chunk_num == 1 offsets, rb = ud.next_offsets() assert rb is None assert offsets.chunk_num == 1 assert offsets.num_bytes == 16 assert offsets.range_start == 16 assert offsets.range_end == 31 assert offsets.pad assert ud._offset == 32 assert ud._chunk_num == 2
def test_resume(tmpdir): tmpdir.join('a').write('zz') lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 0 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None # test no resume ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), None) assert ud._resume() is None # check if path exists in resume db resume = mock.MagicMock() resume.get_record.return_value = None ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume) assert ud._resume() is None # check same lengths bad = mock.MagicMock() bad.length = 0 resume.get_record.return_value = bad assert ud._resume() is None # check completed resume comp = mock.MagicMock() comp.length = 2 comp.completed = True comp.total_chunks = 1 comp.chunk_size = 2 comp.completed_chunks = 1 resume.get_record.return_value = comp ud._completed_chunks = mock.MagicMock() ud._src_ase = ase assert ud._resume() == 2 ase.replica_targets = [ase] ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume) ud._completed_chunks = mock.MagicMock() ud._src_ase = ase assert ud._resume() == 4 # check no encryption ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' opts.rsa_public_key = 'abc' nc = mock.MagicMock() nc.length = 16 nc.completed = False nc.total_chunks = 2 nc.chunk_size = 1 nc.completed_chunks = 1 resume.get_record.return_value = nc ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume) assert ud._resume() is None # check rr path exists ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' nc.length = 2 nc.local_path = pathlib.Path('yyy') opts.rsa_public_key = None resume.get_record.return_value = nc ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume) assert ud._resume() is None # check resume no md5 opts.store_file_properties.md5 = False ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' nc = mock.MagicMock() nc.length = 2 nc.completed = False nc.total_chunks = 2 nc.chunk_size = 1 cc = bitstring.BitArray(length=nc.total_chunks) cc.set(True, 0) nc.completed_chunks = cc.int nc.local_path = lp.absolute_path resume.get_record.return_value = nc ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume) assert ud._resume() == 1 # check resume with md5 mismatch opts.store_file_properties.md5 = True ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' nc = mock.MagicMock() nc.length = 2 nc.completed = False nc.total_chunks = 2 nc.chunk_size = 1 cc = bitstring.BitArray(length=nc.total_chunks) cc.set(True, 0) nc.completed_chunks = cc.int nc.local_path = lp.absolute_path resume.get_record.return_value = nc ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume) assert ud._resume() is None # check resume with md5 match ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' nc = mock.MagicMock() nc.length = 2 nc.completed = False nc.total_chunks = 2 nc.chunk_size = 1 cc = bitstring.BitArray(length=nc.total_chunks) cc.set(True, 0) nc.completed_chunks = cc.int nc.local_path = lp.absolute_path md5 = hashlib.md5() md5.update(b'z') nc.md5hexdigest = md5.hexdigest() resume.get_record.return_value = nc ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume) assert ud._resume() == 1
def test_descriptor_adjust_chunk_size(tmpdir): tmpdir.join('a').ensure(file=True) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) opts = mock.MagicMock() opts.chunk_size_bytes = 0 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud._chunk_size == 0 with mock.patch('blobxfer.models.upload._DEFAULT_AUTO_CHUNKSIZE_BYTES', 1): with mock.patch( 'blobxfer.models.upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES', 3): with mock.patch('blobxfer.models.upload._MAX_NUM_CHUNKS', 2): tmpdir.join('a').write('z' * 4) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud._chunk_size == 2 lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('-'), use_stdin=True) ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud._chunk_size == upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES tmpdir.join('a').write('z' * 32) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Page ase._name = 'name' ase._encryption = None ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud._chunk_size == 32 ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Append ase._name = 'name' ase._encryption = None opts.chunk_size_bytes = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES + 1 with mock.patch( 'blobxfer.models.upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES', 4): ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud._chunk_size == 4 ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None opts.chunk_size_bytes = 32 opts.one_shot_bytes = 32 ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud._chunk_size == 32 opts.one_shot_bytes = 31 with mock.patch('blobxfer.models.upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES', 4): ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud._chunk_size == 4 ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.File ase._name = 'name' ase._encryption = None opts.chunk_size_bytes = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES + 1 with mock.patch( 'blobxfer.models.upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES', 4): ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud._chunk_size == 4 ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Page ase._name = 'name' ase._encryption = None opts.chunk_size_bytes = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES + 1 with mock.patch( 'blobxfer.models.upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES', 4): ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) assert ud._chunk_size == 4 with mock.patch('blobxfer.models.upload._MAX_PAGE_BLOB_SIZE', 4): with pytest.raises(RuntimeError): upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock())
def test_descriptor_compute_remote_size(tmpdir): tmpdir.join('a').write('z' * 32) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) # encrypted remote size with replica opts = mock.MagicMock() opts.chunk_size_bytes = 16 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = 'abc' ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = 'abc' ase2 = azmodels.StorageEntity('cont') ase2._mode = azmodels.StorageModes.Block ase2._name = 'name2' ase.replica_targets = [ase2] ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._compute_remote_size(opts) assert ud.entity.size == 48 for rt in ase.replica_targets: assert rt.size == ud.entity.size # remote size opts = mock.MagicMock() opts.chunk_size_bytes = 16 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._compute_remote_size(opts) assert ud.entity.size == 32 # remote size of zero tmpdir.join('b').ensure(file=True) lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('b')) ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._compute_remote_size(opts) assert ud.entity.size == 0 # stdin as page, resize lp = upload.LocalPath(pathlib.Path('-'), pathlib.Path('-'), use_stdin=True) opts.stdin_as_page_blob_size = 0 ase._mode = azmodels.StorageModes.Page ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._compute_remote_size(opts) assert ud.entity.size == upload._MAX_PAGE_BLOB_SIZE assert ud._needs_resize # stdin as page, no resize lp = upload.LocalPath(pathlib.Path('-'), pathlib.Path('-'), use_stdin=True) opts.stdin_as_page_blob_size = 32 ase._mode = azmodels.StorageModes.Page ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._compute_remote_size(opts) assert ud.entity.size == 32 assert not ud._needs_resize
def test_descriptor_generate_metadata(tmpdir): tmpdir.join('a').write('ab') lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) # test nothing opts = mock.MagicMock() opts.chunk_size_bytes = 1 opts.one_shot_bytes = 0 opts.store_file_properties.attributes = False opts.store_file_properties.md5 = False opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) meta = ud.generate_metadata() assert meta is None # test page md5 align opts = mock.MagicMock() opts.chunk_size_bytes = 1 opts.one_shot_bytes = 0 opts.store_file_properties.attributes = False opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Page ase._name = 'name' ase._encryption = None ase._size = 1 ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._offset = 1 ud.md5 = hashlib.md5() ud.md5.update(b'z') meta = ud.generate_metadata() assert meta is None md5 = hashlib.md5() md5.update(b'z' + b'\0' * 511) assert ud.md5.digest() == md5.digest() # test fileattr meta opts = mock.MagicMock() opts.chunk_size_bytes = 1 opts.one_shot_bytes = 0 opts.store_file_properties.attributes = True opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None # file attr store is not avail on windows if not util.on_windows(): ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) meta = ud.generate_metadata() assert metadata.JSON_KEY_BLOBXFER_METADATA in meta assert metadata._JSON_KEY_FILE_ATTRIBUTES in meta[ metadata.JSON_KEY_BLOBXFER_METADATA] # test enc meta opts.store_file_properties.attributes = False opts.store_file_properties.md5 = False opts.rsa_public_key = 'abc' ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ase.encryption_metadata = mock.MagicMock() ase.encryption_metadata.convert_to_json_with_mac.return_value = { 'encmeta': 'encmeta' } meta = ud.generate_metadata() assert 'encmeta' in meta ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud.hmac = None ase.encryption_metadata = mock.MagicMock() ase.encryption_metadata.convert_to_json_with_mac.return_value = { 'encmeta': 'encmeta' } meta = ud.generate_metadata() assert 'encmeta' in meta opts.store_file_properties.md5 = True ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ase.encryption_metadata = mock.MagicMock() ase.encryption_metadata.convert_to_json_with_mac.return_value = { 'encmeta': 'encmeta' } meta = ud.generate_metadata() assert 'encmeta' in meta # test vio meta opts = mock.MagicMock() opts.chunk_size_bytes = 1 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None lp.view = mock.MagicMock() lp.view.mode = upload.VectoredIoDistributionMode.Stripe ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) with mock.patch( 'blobxfer.models.metadata.generate_vectored_io_stripe_metadata', return_value={'viometa': 'viometa'}): meta = ud.generate_metadata() assert metadata.JSON_KEY_BLOBXFER_METADATA in meta assert 'viometa' in meta[metadata.JSON_KEY_BLOBXFER_METADATA]
def test_descriptor_read_data(tmpdir): tmpdir.join('a').write('ab') lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a')) # test normal opts = mock.MagicMock() opts.chunk_size_bytes = 1 opts.one_shot_bytes = 0 opts.store_file_properties.md5 = True opts.rsa_public_key = None ase = azmodels.StorageEntity('cont') ase._mode = azmodels.StorageModes.Block ase._name = 'name' ase._encryption = None ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), mock.MagicMock()) ud._resume = mock.MagicMock() ud._resume.return_value = None # test no data to read mockoffsets = mock.MagicMock() mockoffsets.num_bytes = 0 data, newoffset = ud.read_data(mockoffsets) assert data is None assert newoffset is None # test normal data to read offsets, rb = ud.next_offsets() assert rb is None data, newoffset = ud.read_data(offsets) assert data == b'a' assert newoffset is None # test stdin with mock.patch('blobxfer.STDIN', new_callable=mock.PropertyMock) as patched_stdin: patched_stdin.read = mock.MagicMock() patched_stdin.read.return_value = b'z' ud.local_path.use_stdin = True data, newoffset = ud.read_data(offsets) assert data == b'z' assert newoffset.chunk_num == 0 assert newoffset.num_bytes == 1 assert newoffset.range_start == 0 assert newoffset.range_end == 0 assert not newoffset.pad assert ud._total_chunks == 3 assert ud._outstanding_ops == 3 assert ud._offset == 1 assert ud.entity.size == 2 with mock.patch('blobxfer.STDIN', new_callable=mock.PropertyMock) as patched_stdin: patched_stdin.read = mock.MagicMock() patched_stdin.read.return_value = None ud.local_path.use_stdin = True data, newoffset = ud.read_data(offsets) assert data is None assert newoffset is None assert ud._total_chunks == 2 assert ud._outstanding_ops == 2 assert ud._chunk_num == 0