def test_cleanup_all_temporary_files(tmpdir):
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 16
lp = pathlib.Path(str(tmpdir.join('a')))
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
data = b'0' * opts.chunk_size_bytes
d.write_unchecked_data(offsets, data)
assert len(d._unchecked_chunks) == 1
d.cleanup_all_temporary_files()
assert not d.final_path.exists()
assert not d._unchecked_chunks[0]['ucc'].file_path.exists()
lp = pathlib.Path(str(tmpdir.join('b')))
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
data = b'0' * opts.chunk_size_bytes
d.write_unchecked_hmac_data(offsets, data)
assert len(d._unchecked_chunks) == 1
d._unchecked_chunks[0]['ucc'].file_path.unlink()
d.cleanup_all_temporary_files()
assert not d.final_path.exists()
assert not d._unchecked_chunks[0]['ucc'].file_path.exists()
# go through except path
d.cleanup_all_temporary_files()
assert not d.final_path.exists()
def test_downloaddescriptor(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 1024
ase._encryption = mock.MagicMock()
with pytest.raises(RuntimeError):
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
ase._encryption.symmetric_key = b'123'
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
assert not d._allocated
d._allocate_disk_space()
assert d.entity == ase
assert d.entity.is_encrypted
assert not d.must_compute_md5
assert d.hmac is not None
assert d._total_chunks == 64
assert d._offset == 0
assert d.final_path == lp
assert d._allocated
assert d.final_path.stat().st_size == ase._size - 16
d._allocate_disk_space()
assert d._allocated
d.final_path.unlink()
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
assert d._total_chunks == 2
assert d._allocated
assert d.final_path.stat().st_size == ase._size - 16
d.final_path.unlink()
ase._encryption = None
ase._size = 1024
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
assert d._allocated
assert d.final_path.stat().st_size == ase._size
# pre-existing file check
opts.chunk_size_bytes = 0
ase._size = 0
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
assert d._total_chunks == 0
assert d._allocated
assert d.final_path.stat().st_size == ase._size
def test_cleanup_temporary_files(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 16
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd._allocate_disk_space()
dd.cleanup_all_temporary_files = mock.MagicMock()
dd.cleanup_all_temporary_files.side_effect = Exception
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.resume_file = pathlib.Path('abc')
d._dd_map[0] = dd
d._cleanup_temporary_files()
assert dd.final_path.exists()
lp = pathlib.Path(str(tmpdir.join('b')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 16
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd._allocate_disk_space()
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.resume_file = None
d._dd_map[0] = dd
d._cleanup_temporary_files()
assert not dd.final_path.exists()
lp = pathlib.Path(str(tmpdir.join('c')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 16
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd._allocate_disk_space()
dd.cleanup_all_temporary_files = mock.MagicMock()
dd.cleanup_all_temporary_files.side_effect = Exception
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.resume_file = None
d._dd_map[0] = dd
d._cleanup_temporary_files()
assert dd.final_path.exists()
def test_mark_unchecked_chunk_decrypted():
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 32
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(mock.MagicMock(), ase, opts, mock.MagicMock(), None)
d._unchecked_chunks[0] = {'decrypted': False}
d.mark_unchecked_chunk_decrypted(0)
assert d._unchecked_chunks[0]
def test_set_final_path_view():
lp = pathlib.Path('/local/path/abc.bxslice-0')
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 1024
ase._vio = mock.MagicMock()
ase._vio.slice_id = 0
ase._vio.total_size = 1024
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
total_size = d._set_final_path_view()
assert total_size == ase._size
def test_operations(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._outstanding_ops = 1
d._unchecked_chunks = {0: None}
assert not d.all_operations_completed
d._outstanding_ops -= 1
d._unchecked_chunks.pop(0)
assert d.all_operations_completed
def test_write_data(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
data = b'0' * ase._size
d.write_data(offsets, data)
assert d.final_path.exists()
assert d.final_path.stat().st_size == len(data)
def test_update_resume_for_completed(tmpdir):
resumefile = pathlib.Path(str(tmpdir.join('resume')))
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._name = 'blob'
ase._client = mock.MagicMock()
rmgr = rops.DownloadResumeManager(resumefile)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
offsets, _ = d.next_offsets()
d._update_resume_for_completed()
dr = rmgr.get_record(ase)
assert dr.completed
def test_hmac_iv(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 256
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
ase._size = 128
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
iv = b'abc'
d.hmac_iv(iv)
assert d.hmac.update.call_count == 1
def test_downloaddescriptor_allocate_disk_space_via_seek(tmpdir):
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 256
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), None)
with mock.patch('os.posix_fallocate') as patched_fallocate:
patched_fallocate.side_effect = [AttributeError()]
d._allocate_disk_space()
assert d._allocated
assert fp.exists()
assert fp.stat().st_size == ase._size
def test_restore_file_attributes(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
lp.touch(mode=0o666, exist_ok=False)
lp.exists()
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._fileattr = mock.MagicMock()
ase._fileattr.mode = '0o100777'
ase._fileattr.uid = 1000
ase._fileattr.gid = 1000
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._restore_file_attributes()
stat = lp.stat()
assert str(oct(stat.st_mode)).replace('o', '') == \
ase._fileattr.mode.replace('o', '')
def test_write_unchecked_hmac_data(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 32
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
d.write_unchecked_hmac_data(offsets, b'0' * ase._size)
assert offsets.chunk_num in d._unchecked_chunks
ucc = d._unchecked_chunks[offsets.chunk_num]
assert ucc['ucc'].data_len == ase._size
assert ucc['ucc'].fd_start == offsets.fd_start
assert ucc['ucc'].file_path != d.final_path
assert ucc['ucc'].temp
assert not ucc['decrypted']
def test_process_download_descriptor_vio(tmpdir):
with mock.patch(
'blobxfer.models.download.Descriptor.all_operations_completed',
new_callable=mock.PropertyMock) as patched_aoc:
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(),
mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.File
ase._size = 16
ase._client = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase._name = 'name'
ase._vio = mock.MagicMock()
ase._vio.total_slices = 2
lp = pathlib.Path(str(tmpdir.join('b')))
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd.next_offsets = mock.MagicMock()
dd.next_offsets.return_value = (None, None)
patched_aoc.return_value = True
dd.finalize_file = mock.MagicMock()
key = ops.Downloader.create_unique_transfer_operation_id(ase)
d._transfer_set.add(key)
d._dd_map[str(lp)] = mock.MagicMock()
d._transfer_cc[dd.final_path] = mock.MagicMock()
d._process_download_descriptor(dd)
assert dd.finalize_file.call_count == 0
d._transfer_set.add(key)
d._dd_map[str(lp)] = mock.MagicMock()
d._transfer_cc[dd.final_path] = mock.MagicMock()
d._process_download_descriptor(dd)
assert dd.finalize_file.call_count == 1
def test_restore_file_lmt(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
lp.touch(mode=0o666, exist_ok=False)
lp.exists()
ts = util.datetime_now() - datetime.timedelta(seconds=60)
ts_posix = time.mktime(ts.timetuple())
stat = lp.stat()
assert stat.st_mtime != ts_posix
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
opts.restore_file_properties.lmt = True
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._lmt = ts
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._restore_file_lmt()
stat = lp.stat()
assert stat.st_mtime == ts_posix
def test_worker_thread_transfer(
patched_gbr, patched_gfr, patched_acdd, tmpdir):
# test disk set > max set length
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(
mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._process_download_descriptor = mock.MagicMock()
d._general_options.concurrency.disk_threads = 1
d._disk_set.add(0)
d._disk_set.add(1)
d._disk_set.add(2)
d._disk_set.add(3)
d._disk_set.add(4)
patched_tc.side_effect = [False, True]
d._worker_thread_transfer()
assert d._process_download_descriptor.call_count == 0
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._process_download_descriptor = mock.MagicMock()
d._download_terminate = True
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
d._worker_thread_transfer()
assert d._process_download_descriptor.call_count == 0
d._download_terminate = False
d._all_remote_files_processed = True
d._worker_thread_transfer()
assert d._process_download_descriptor.call_count == 0
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
patched_tc.side_effect = [False, False, True]
ase = azmodels.StorageEntity('cont')
ase._size = 16
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'abc'
lp = pathlib.Path(str(tmpdir.join('exc')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._transfer_queue = mock.MagicMock()
d._transfer_queue.get.side_effect = [queue.Empty, dd]
d._process_download_descriptor = mock.MagicMock()
d._process_download_descriptor.side_effect = RuntimeError('oops')
d._worker_thread_transfer()
assert len(d._exceptions) == 1
assert d._process_download_descriptor.call_count == 1
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
with mock.patch(
'blobxfer.models.download.Descriptor.'
'all_operations_completed',
new_callable=mock.PropertyMock) as patched_aoc:
d = ops.Downloader(
mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 16
ase._client = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase._name = 'name'
ase._vio = None
key = ops.Downloader.create_unique_transfer_operation_id(ase)
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'abc'
lp = pathlib.Path(str(tmpdir.join('a')))
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd.next_offsets = mock.MagicMock(
side_effect=[(None, 1), (None, 2)])
dd.finalize_integrity = mock.MagicMock()
dd.finalize_file = mock.MagicMock()
dd.perform_chunked_integrity_check = mock.MagicMock()
dd.all_operations_completed.side_effect = [False, True]
patched_aoc.side_effect = [False, True]
patched_tc.side_effect = [False, False, False, True]
d._dd_map[str(lp)] = dd
d._transfer_set.add(key)
d._transfer_queue = mock.MagicMock()
d._transfer_queue.get.side_effect = [queue.Empty, dd, dd]
d._worker_thread_transfer()
assert str(lp) not in d._dd_map
assert dd.finalize_file.call_count == 1
assert d._download_sofar == 1
assert d._download_bytes_sofar == 3
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(
mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.File
ase._size = 16
ase._client = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase._name = 'name'
ase._vio = None
key = ops.Downloader.create_unique_transfer_operation_id(ase)
patched_gfr.return_value = b'0' * ase._size
lp = pathlib.Path(str(tmpdir.join('b')))
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd.finalize_file = mock.MagicMock()
dd.perform_chunked_integrity_check = mock.MagicMock()
d._dd_map[str(lp)] = mock.MagicMock()
d._transfer_cc[dd.entity.path] = 0
d._transfer_set.add(key)
d._transfer_queue = mock.MagicMock()
d._transfer_queue.get.side_effect = [dd]
patched_tc.side_effect = [False, True]
d._spec.options.max_single_object_concurrency = 0
d._worker_thread_transfer()
assert len(d._disk_set) == 1
a, b, c = d._disk_queue.get()
d._process_data(a, b, c)
assert dd.perform_chunked_integrity_check.call_count == 1
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(
mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
d._spec.options.max_single_object_concurrency = 8
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Auto
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'abc'
ase._encryption.content_encryption_iv = b'0' * 16
ase._client = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase._name = 'name'
ase._vio = None
key = ops.Downloader.create_unique_transfer_operation_id(ase)
patched_gfr.return_value = b'0' * ase._size
lp = pathlib.Path(str(tmpdir.join('c')))
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd.finalize_file = mock.MagicMock()
dd.write_unchecked_hmac_data = mock.MagicMock()
dd.perform_chunked_integrity_check = mock.MagicMock()
d._crypto_offload = mock.MagicMock()
d._crypto_offload.add_decrypt_chunk = mock.MagicMock()
d._dd_map[str(lp)] = dd
d._transfer_cc[dd.entity.path] = 0
d._transfer_set.add(key)
d._transfer_queue = mock.MagicMock()
d._transfer_queue.get.side_effect = [dd]
patched_tc.side_effect = [False, True]
d._worker_thread_transfer()
assert len(d._disk_set) == 1
a, b, c = d._disk_queue.get()
d._process_data(a, b, c)
assert d._crypto_offload.add_decrypt_chunk.call_count == 1
assert dd.write_unchecked_hmac_data.call_count == 1
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(
mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.crypto_processes = 0
d._general_options.concurrency.transfer_threads = 1
d._general_options.concurrency.disk_threads = 1
d._spec.options.max_single_object_concurrency = 8
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Auto
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'abc'
ase._encryption.content_encryption_iv = b'0' * 16
ase._client = mock.MagicMock()
ase._client.primary_endpoint = 'ep'
ase._name = 'name'
ase._vio = None
key = ops.Downloader.create_unique_transfer_operation_id(ase)
patched_gfr.return_value = b'0' * ase._size
lp = pathlib.Path(str(tmpdir.join('d')))
dd = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
dd.next_offsets()
dd.write_unchecked_hmac_data = mock.MagicMock()
dd.perform_chunked_integrity_check = mock.MagicMock()
dd.mark_unchecked_chunk_decrypted = mock.MagicMock()
patched_acdd.return_value = b'0' * 16
d._dd_map[str(lp)] = mock.MagicMock()
d._transfer_cc[dd.entity.path] = 0
d._transfer_set.add(key)
d._transfer_queue = mock.MagicMock()
d._transfer_queue.get.side_effect = [dd, dd]
patched_tc.side_effect = [False, True]
d._worker_thread_transfer()
assert len(d._disk_set) == 1
a, b, c = d._disk_queue.get()
d._process_data(a, b, c)
assert patched_acdd.call_count == 1
assert dd.write_unchecked_hmac_data.call_count == 1
assert dd.perform_chunked_integrity_check.call_count == 1
def test_finalize_integrity_and_file(tmpdir):
# already finalized
lp = pathlib.Path(str(tmpdir.join('af')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
data = b'0' * ase._size
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
d._finalized = True
d.finalize_integrity()
d.finalize_file()
assert d.final_path.exists()
assert d.final_path.stat().st_size == ase._size
d.final_path.unlink()
# hmac check success
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
signkey = os.urandom(32)
ase._encryption.initialize_hmac = mock.MagicMock()
ase._encryption.initialize_hmac.return_value = hmac.new(
signkey, digestmod=hashlib.sha256)
data = b'0' * (ase._size - 16)
_hmac = hmac.new(signkey, digestmod=hashlib.sha256)
_hmac.update(data)
ase._encryption.encryption_authentication.\
message_authentication_code = util.base64_encode_as_string(
_hmac.digest())
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
d.hmac.update(data)
d.finalize_integrity()
d.finalize_file()
assert d.final_path.exists()
assert d.final_path.stat().st_size == len(data)
# md5 check success
lp = pathlib.Path(str(tmpdir.join('b')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
data = b'0' * ase._size
md5 = util.new_md5_hasher()
md5.update(data)
ase._md5 = util.base64_encode_as_string(md5.digest())
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
d.md5.update(data)
d.finalize_integrity()
d.finalize_file()
assert d.final_path.exists()
assert d.final_path.stat().st_size == len(data)
# no check
lp = pathlib.Path(str(tmpdir.join('c')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
data = b'0' * ase._size
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
d.finalize_integrity()
d.finalize_file()
assert d.final_path.exists()
assert d.final_path.stat().st_size == len(data)
# md5 mismatch
lp = pathlib.Path(str(tmpdir.join('d')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
data = b'0' * ase._size
ase._md5 = 'oops'
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
d.md5.update(data)
d.finalize_integrity()
d.finalize_file()
assert not d.final_path.exists()
def test_perform_chunked_integrity_check(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
data = b'0' * opts.chunk_size_bytes
d.write_unchecked_data(offsets, data)
d.perform_chunked_integrity_check()
assert d._next_integrity_chunk == 1
assert 0 not in d._unchecked_chunks
assert len(d._unchecked_chunks) == 0
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
data = b'0' * opts.chunk_size_bytes
offsets, _ = d.next_offsets()
d.write_unchecked_hmac_data(offsets, data)
ucc = d._unchecked_chunks[offsets.chunk_num]
offsets1, _ = d.next_offsets()
d.write_unchecked_hmac_data(offsets1, data)
ucc1 = d._unchecked_chunks[offsets1.chunk_num]
ucc['decrypted'] = True
ucc1['decrypted'] = True
d.perform_chunked_integrity_check()
assert ucc['ucc'].file_path != d.final_path
assert ucc1['ucc'].file_path != d.final_path
assert d._next_integrity_chunk == 2
assert 0 not in d._unchecked_chunks
assert 1 not in d._unchecked_chunks
assert len(d._unchecked_chunks) == 0
# check integrity with resume
resumefile = pathlib.Path(str(tmpdir.join('resume')))
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
data = b'0' * opts.chunk_size_bytes
md5 = util.new_md5_hasher()
md5.update(data)
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._name = 'blob'
ase._client = mock.MagicMock()
ase._md5 = md5.hexdigest()
rmgr = rops.DownloadResumeManager(resumefile)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
offsets, _ = d.next_offsets()
d.write_unchecked_data(offsets, data)
d.perform_chunked_integrity_check()
assert d._next_integrity_chunk == 1
assert len(d._unchecked_chunks) == 0
dr = rmgr.get_record(ase)
assert dr.next_integrity_chunk == 1
assert dr.md5hexdigest == md5.hexdigest()
def test_downloaddescriptor_next_offsets(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 256
ase = azmodels.StorageEntity('cont')
ase._size = 128
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, resume_bytes = d.next_offsets()
assert resume_bytes is None
assert d._total_chunks == 1
assert offsets.chunk_num == 0
assert offsets.fd_start == 0
assert offsets.num_bytes == 128
assert offsets.range_start == 0
assert offsets.range_end == 127
assert not offsets.unpad
assert d.next_offsets() == (None, None)
ase._size = 0
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
assert d._total_chunks == 0
assert d.next_offsets() == (None, None)
ase._size = 1
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, resume_bytes = d.next_offsets()
assert resume_bytes is None
assert d._total_chunks == 1
assert offsets.chunk_num == 0
assert offsets.fd_start == 0
assert offsets.num_bytes == 1
assert offsets.range_start == 0
assert offsets.range_end == 0
assert not offsets.unpad
assert d.next_offsets() == (None, None)
ase._size = 256
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, resume_bytes = d.next_offsets()
assert resume_bytes is None
assert d._total_chunks == 1
assert offsets.chunk_num == 0
assert offsets.fd_start == 0
assert offsets.num_bytes == 256
assert offsets.range_start == 0
assert offsets.range_end == 255
assert not offsets.unpad
assert d.next_offsets() == (None, None)
ase._size = 256 + 16
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, resume_bytes = d.next_offsets()
assert resume_bytes is None
assert d._total_chunks == 2
assert offsets.chunk_num == 0
assert offsets.fd_start == 0
assert offsets.num_bytes == 256
assert offsets.range_start == 0
assert offsets.range_end == 255
assert not offsets.unpad
offsets, resume_bytes = d.next_offsets()
assert resume_bytes is None
assert offsets.chunk_num == 1
assert offsets.fd_start == 256
assert offsets.num_bytes == 16
assert offsets.range_start == 256
assert offsets.range_end == 256 + 15
assert not offsets.unpad
assert d.next_offsets() == (None, None)
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
ase._size = 128
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, resume_bytes = d.next_offsets()
assert resume_bytes is None
assert d._total_chunks == 1
assert offsets.chunk_num == 0
assert offsets.fd_start == 0
assert offsets.num_bytes == 128
assert offsets.range_start == 0
assert offsets.range_end == 127
assert offsets.unpad
assert d.next_offsets() == (None, None)
ase._size = 256
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, resume_bytes = d.next_offsets()
assert resume_bytes is None
assert d._total_chunks == 1
assert offsets.chunk_num == 0
assert offsets.fd_start == 0
assert offsets.num_bytes == 256
assert offsets.range_start == 0
assert offsets.range_end == 255
assert offsets.unpad
assert d.next_offsets() == (None, None)
ase._size = 256 + 32 # 16 bytes over + padding
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, resume_bytes = d.next_offsets()
assert resume_bytes is None
assert d._total_chunks == 2
assert offsets.chunk_num == 0
assert offsets.fd_start == 0
assert offsets.num_bytes == 256
assert offsets.range_start == 0
assert offsets.range_end == 255
assert not offsets.unpad
offsets, resume_bytes = d.next_offsets()
assert resume_bytes is None
assert offsets.chunk_num == 1
assert offsets.fd_start == 256
assert offsets.num_bytes == 32
assert offsets.range_start == 256 - 16
assert offsets.range_end == 256 + 31
assert offsets.unpad
assert d.next_offsets() == (None, None)
def test_downloaddescriptor_resume(tmpdir):
resumefile = pathlib.Path(str(tmpdir.join('resume')))
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 256
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
# test no record
rmgr = rops.DownloadResumeManager(resumefile)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# test length mismatch
rmgr.add_or_update_record(str(fp), ase, 0, 0, False, None)
ase._size = 127
rb = d._resume()
assert rb is None
ase._size = 128
# test nothing to resume
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
rmgr.add_or_update_record(str(fp), ase, 0, 0, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# test completion
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
rmgr.add_or_update_record(str(fp), ase, 32, 1, True, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
fp.touch()
rb = d._resume()
assert rb == ase._size
# test encrypted no resume
fp.unlink()
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# test up to chunk
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb == 32
# ensure hmac not populated
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
fp.touch()
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
d.hmac = True
with pytest.raises(RuntimeError):
d._resume()
# md5 hash check
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
data = os.urandom(32)
with fp.open('wb') as f:
f.write(data)
md5 = util.new_md5_hasher()
md5.update(data)
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, md5.hexdigest())
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb == 32
# md5 hash mismatch
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, 'abc')
ase._md5 = 'abc'
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# md5 hash check as page file
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
ase._mode = azmodels.StorageModes.Page
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, md5.hexdigest())
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb == 32
