def test_compute_total_chunks(tmpdir):
tmpdir.join('a').ensure(file=True)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 0
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES
with pytest.raises(RuntimeError):
ud._compute_total_chunks(1)
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES
ud._chunk_size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES
with pytest.raises(RuntimeError):
ud._compute_total_chunks(1)
ase._mode = azmodels.StorageModes.Append
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud.entity.size = upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES
ud._chunk_size = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES
with pytest.raises(RuntimeError):
ud._compute_total_chunks(1)
def test_descriptor(tmpdir):
size = 32
tmpdir.join('a').write('z' * size)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 8
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = False
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._size = size
ase._encryption = None
ase2 = azmodels.StorageEntity('cont')
ase2._mode = azmodels.StorageModes.Block
ase2._name = 'name2'
ase2._size = size
ase2._encryption = None
ase.replica_targets = [ase2]
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud.hmac is None
assert ud.md5 is None
assert ud._outstanding_ops == 4 * 2
assert ud._completed_chunks is not None
assert ud._md5_cache is not None
assert ud._replica_counters is not None
assert ud.entity == ase
assert not ud.must_compute_md5
assert not ud.all_operations_completed
assert ud.last_block_num == -1
assert ud.is_resumable
assert not ud.remote_is_file
assert not ud.remote_is_page_blob
assert not ud.remote_is_append_blob
assert not ud.is_one_shot_block_blob
assert ud.requires_put_block_list
assert not ud.requires_non_encrypted_md5_put
assert not ud.requires_set_file_properties_md5
assert not ud.requires_access_tier_set
assert ud.requires_resize() == (False, ud._offset)
# test sym key
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._size = size
ase._encryption = mock.MagicMock()
opts.rsa_public_key = None
with pytest.raises(RuntimeError):
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
def test_localpath(tmpdir):
tmpdir.join('a').write('zz')
pp = pathlib.Path(str(tmpdir))
rp = pathlib.Path('a')
file = pp / rp
stat = file.stat()
lp = upload.LocalPath(pp, rp, use_stdin=True, view=None)
assert lp.absolute_path == file
assert lp.size == 0
assert lp.total_size == 0
assert lp.lmt == 0
assert lp.mode.replace('o', '') == '00'
assert lp.uid == 0
assert lp.gid == 0
lp = upload.LocalPath(pp, rp, use_stdin=False, view=None)
assert lp.absolute_path == file
assert lp.size == stat.st_size
assert lp.total_size == stat.st_size
assert lp.lmt == stat.st_mtime
assert lp.mode.replace('o', '') == str(oct(stat.st_mode)).replace('o', '')
assert lp.uid == stat.st_uid
assert lp.gid == stat.st_gid
lpview = upload.LocalPathView(
fd_start=1,
fd_end=2,
slice_num=1,
mode=upload.VectoredIoDistributionMode.Stripe,
total_slices=2,
next=None,
)
lp = upload.LocalPath(pp, rp, use_stdin=False, view=lpview)
assert lp.absolute_path == file
assert lp.size == 1
assert lp.total_size == stat.st_size
assert lp.lmt == stat.st_mtime
assert lp.mode.replace('o', '') == str(oct(stat.st_mode)).replace('o', '')
assert lp.uid == stat.st_uid
assert lp.gid == stat.st_gid
def test_descriptor_initialize_encryption(tmpdir):
tmpdir.join('a').write('z' * 32)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 16
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = 'abc'
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._size = 32
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud.hmac is not None
assert ud.entity.is_encrypted
def test_descriptor_complete_offset_upload(tmpdir):
tmpdir.join('a').write('z' * 32)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 16
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._size = 32
ase._encryption = None
ase.replica_targets = [ase]
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._md5_cache[0] = 'md50'
ud._md5_cache[1] = 'md51'
ud.complete_offset_upload(0)
assert ud._outstanding_ops == 3
assert ud._replica_counters[0] == 0
ud.complete_offset_upload(1)
assert ud._outstanding_ops == 2
assert ud._replica_counters[1] == 0
# fill md5 cache with junk to trigger gc on next complete
for i in range(-30, -1):
ud._md5_cache[i] = ''
ud.complete_offset_upload(0)
assert ud._outstanding_ops == 1
assert 0 not in ud._replica_counters
assert len(ud._md5_cache) == 2
ud.complete_offset_upload(1)
assert ud._outstanding_ops == 0
assert 1 not in ud._replica_counters
assert len(ud._md5_cache) == 0
def test_descriptor_hmac_data(tmpdir):
tmpdir.join('a').write('z' * 32)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 16
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = 'abc'
ase.replica_targets = [ase]
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud.hmac is not None
ud.hmac_data(b'\0')
def test_descriptor_next_offsets(tmpdir):
tmpdir.join('a').write('ab')
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
# test normal
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._resume = mock.MagicMock()
ud._resume.return_value = None
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets.chunk_num == 0
assert offsets.num_bytes == 1
assert offsets.range_start == 0
assert offsets.range_end == 0
assert not offsets.pad
assert ud._offset == 1
assert ud._chunk_num == 1
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets.chunk_num == 1
assert offsets.num_bytes == 1
assert offsets.range_start == 1
assert offsets.range_end == 1
assert not offsets.pad
assert ud._offset == 2
assert ud._chunk_num == 2
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets is None
# test chunk size exceeds size
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts.chunk_size_bytes = 3
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._chunk_size = 3
ud._resume = mock.MagicMock()
ud._resume.return_value = None
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets.chunk_num == 0
assert offsets.num_bytes == 2
assert offsets.range_start == 0
assert offsets.range_end == 1
assert not offsets.pad
assert ud._offset == 2
assert ud._chunk_num == 1
# test encrypted
tmpdir.join('a').write('z' * 16)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts.chunk_size_bytes = 16
opts.rsa_public_key = 'abc'
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._resume = mock.MagicMock()
ud._resume.return_value = None
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets.chunk_num == 0
assert offsets.num_bytes == 16
assert offsets.range_start == 0
assert offsets.range_end == 15
assert not offsets.pad
assert ud._offset == 16
assert ud._chunk_num == 1
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets.chunk_num == 1
assert offsets.num_bytes == 16
assert offsets.range_start == 16
assert offsets.range_end == 31
assert offsets.pad
assert ud._offset == 32
assert ud._chunk_num == 2
def test_resume(tmpdir):
tmpdir.join('a').write('zz')
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 0
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
# test no resume
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), None)
assert ud._resume() is None
# check if path exists in resume db
resume = mock.MagicMock()
resume.get_record.return_value = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() is None
# check same lengths
bad = mock.MagicMock()
bad.length = 0
resume.get_record.return_value = bad
assert ud._resume() is None
# check completed resume
comp = mock.MagicMock()
comp.length = 2
comp.completed = True
comp.total_chunks = 1
comp.chunk_size = 2
comp.completed_chunks = 1
resume.get_record.return_value = comp
ud._completed_chunks = mock.MagicMock()
ud._src_ase = ase
assert ud._resume() == 2
ase.replica_targets = [ase]
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
ud._completed_chunks = mock.MagicMock()
ud._src_ase = ase
assert ud._resume() == 4
# check no encryption
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
opts.rsa_public_key = 'abc'
nc = mock.MagicMock()
nc.length = 16
nc.completed = False
nc.total_chunks = 2
nc.chunk_size = 1
nc.completed_chunks = 1
resume.get_record.return_value = nc
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() is None
# check rr path exists
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
nc.length = 2
nc.local_path = pathlib.Path('yyy')
opts.rsa_public_key = None
resume.get_record.return_value = nc
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() is None
# check resume no md5
opts.store_file_properties.md5 = False
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
nc = mock.MagicMock()
nc.length = 2
nc.completed = False
nc.total_chunks = 2
nc.chunk_size = 1
cc = bitstring.BitArray(length=nc.total_chunks)
cc.set(True, 0)
nc.completed_chunks = cc.int
nc.local_path = lp.absolute_path
resume.get_record.return_value = nc
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() == 1
# check resume with md5 mismatch
opts.store_file_properties.md5 = True
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
nc = mock.MagicMock()
nc.length = 2
nc.completed = False
nc.total_chunks = 2
nc.chunk_size = 1
cc = bitstring.BitArray(length=nc.total_chunks)
cc.set(True, 0)
nc.completed_chunks = cc.int
nc.local_path = lp.absolute_path
resume.get_record.return_value = nc
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() is None
# check resume with md5 match
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
nc = mock.MagicMock()
nc.length = 2
nc.completed = False
nc.total_chunks = 2
nc.chunk_size = 1
cc = bitstring.BitArray(length=nc.total_chunks)
cc.set(True, 0)
nc.completed_chunks = cc.int
nc.local_path = lp.absolute_path
md5 = hashlib.md5()
md5.update(b'z')
nc.md5hexdigest = md5.hexdigest()
resume.get_record.return_value = nc
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() == 1
def test_descriptor_adjust_chunk_size(tmpdir):
tmpdir.join('a').ensure(file=True)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 0
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 0
with mock.patch('blobxfer.models.upload._DEFAULT_AUTO_CHUNKSIZE_BYTES', 1):
with mock.patch(
'blobxfer.models.upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES', 3):
with mock.patch('blobxfer.models.upload._MAX_NUM_CHUNKS', 2):
tmpdir.join('a').write('z' * 4)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)),
pathlib.Path('a'))
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 2
lp = upload.LocalPath(pathlib.Path(str(tmpdir)),
pathlib.Path('-'),
use_stdin=True)
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES
tmpdir.join('a').write('z' * 32)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Page
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 32
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Append
ase._name = 'name'
ase._encryption = None
opts.chunk_size_bytes = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES + 1
with mock.patch(
'blobxfer.models.upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES', 4):
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 4
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
opts.chunk_size_bytes = 32
opts.one_shot_bytes = 32
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 32
opts.one_shot_bytes = 31
with mock.patch('blobxfer.models.upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES',
4):
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 4
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.File
ase._name = 'name'
ase._encryption = None
opts.chunk_size_bytes = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES + 1
with mock.patch(
'blobxfer.models.upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES', 4):
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 4
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Page
ase._name = 'name'
ase._encryption = None
opts.chunk_size_bytes = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES + 1
with mock.patch(
'blobxfer.models.upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES', 4):
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 4
with mock.patch('blobxfer.models.upload._MAX_PAGE_BLOB_SIZE', 4):
with pytest.raises(RuntimeError):
upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
def test_descriptor_compute_remote_size(tmpdir):
tmpdir.join('a').write('z' * 32)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
# encrypted remote size with replica
opts = mock.MagicMock()
opts.chunk_size_bytes = 16
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = 'abc'
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = 'abc'
ase2 = azmodels.StorageEntity('cont')
ase2._mode = azmodels.StorageModes.Block
ase2._name = 'name2'
ase.replica_targets = [ase2]
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._compute_remote_size(opts)
assert ud.entity.size == 48
for rt in ase.replica_targets:
assert rt.size == ud.entity.size
# remote size
opts = mock.MagicMock()
opts.chunk_size_bytes = 16
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._compute_remote_size(opts)
assert ud.entity.size == 32
# remote size of zero
tmpdir.join('b').ensure(file=True)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('b'))
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._compute_remote_size(opts)
assert ud.entity.size == 0
# stdin as page, resize
lp = upload.LocalPath(pathlib.Path('-'), pathlib.Path('-'), use_stdin=True)
opts.stdin_as_page_blob_size = 0
ase._mode = azmodels.StorageModes.Page
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._compute_remote_size(opts)
assert ud.entity.size == upload._MAX_PAGE_BLOB_SIZE
assert ud._needs_resize
# stdin as page, no resize
lp = upload.LocalPath(pathlib.Path('-'), pathlib.Path('-'), use_stdin=True)
opts.stdin_as_page_blob_size = 32
ase._mode = azmodels.StorageModes.Page
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._compute_remote_size(opts)
assert ud.entity.size == 32
assert not ud._needs_resize
def test_descriptor_generate_metadata(tmpdir):
tmpdir.join('a').write('ab')
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
# test nothing
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.attributes = False
opts.store_file_properties.md5 = False
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
meta = ud.generate_metadata()
assert meta is None
# test page md5 align
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.attributes = False
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Page
ase._name = 'name'
ase._encryption = None
ase._size = 1
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._offset = 1
ud.md5 = hashlib.md5()
ud.md5.update(b'z')
meta = ud.generate_metadata()
assert meta is None
md5 = hashlib.md5()
md5.update(b'z' + b'\0' * 511)
assert ud.md5.digest() == md5.digest()
# test fileattr meta
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.attributes = True
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
# file attr store is not avail on windows
if not util.on_windows():
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
meta = ud.generate_metadata()
assert metadata.JSON_KEY_BLOBXFER_METADATA in meta
assert metadata._JSON_KEY_FILE_ATTRIBUTES in meta[
metadata.JSON_KEY_BLOBXFER_METADATA]
# test enc meta
opts.store_file_properties.attributes = False
opts.store_file_properties.md5 = False
opts.rsa_public_key = 'abc'
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ase.encryption_metadata = mock.MagicMock()
ase.encryption_metadata.convert_to_json_with_mac.return_value = {
'encmeta': 'encmeta'
}
meta = ud.generate_metadata()
assert 'encmeta' in meta
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud.hmac = None
ase.encryption_metadata = mock.MagicMock()
ase.encryption_metadata.convert_to_json_with_mac.return_value = {
'encmeta': 'encmeta'
}
meta = ud.generate_metadata()
assert 'encmeta' in meta
opts.store_file_properties.md5 = True
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ase.encryption_metadata = mock.MagicMock()
ase.encryption_metadata.convert_to_json_with_mac.return_value = {
'encmeta': 'encmeta'
}
meta = ud.generate_metadata()
assert 'encmeta' in meta
# test vio meta
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
lp.view = mock.MagicMock()
lp.view.mode = upload.VectoredIoDistributionMode.Stripe
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
with mock.patch(
'blobxfer.models.metadata.generate_vectored_io_stripe_metadata',
return_value={'viometa': 'viometa'}):
meta = ud.generate_metadata()
assert metadata.JSON_KEY_BLOBXFER_METADATA in meta
assert 'viometa' in meta[metadata.JSON_KEY_BLOBXFER_METADATA]
def test_descriptor_read_data(tmpdir):
tmpdir.join('a').write('ab')
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
# test normal
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._resume = mock.MagicMock()
ud._resume.return_value = None
# test no data to read
mockoffsets = mock.MagicMock()
mockoffsets.num_bytes = 0
data, newoffset = ud.read_data(mockoffsets)
assert data is None
assert newoffset is None
# test normal data to read
offsets, rb = ud.next_offsets()
assert rb is None
data, newoffset = ud.read_data(offsets)
assert data == b'a'
assert newoffset is None
# test stdin
with mock.patch('blobxfer.STDIN',
new_callable=mock.PropertyMock) as patched_stdin:
patched_stdin.read = mock.MagicMock()
patched_stdin.read.return_value = b'z'
ud.local_path.use_stdin = True
data, newoffset = ud.read_data(offsets)
assert data == b'z'
assert newoffset.chunk_num == 0
assert newoffset.num_bytes == 1
assert newoffset.range_start == 0
assert newoffset.range_end == 0
assert not newoffset.pad
assert ud._total_chunks == 3
assert ud._outstanding_ops == 3
assert ud._offset == 1
assert ud.entity.size == 2
with mock.patch('blobxfer.STDIN',
new_callable=mock.PropertyMock) as patched_stdin:
patched_stdin.read = mock.MagicMock()
patched_stdin.read.return_value = None
ud.local_path.use_stdin = True
data, newoffset = ud.read_data(offsets)
assert data is None
assert newoffset is None
assert ud._total_chunks == 2
assert ud._outstanding_ops == 2
assert ud._chunk_num == 0