def test_perform_chunked_integrity_check(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
data = b'0' * opts.chunk_size_bytes
d.write_unchecked_data(offsets, data)
d.perform_chunked_integrity_check()
assert d._next_integrity_chunk == 1
assert 0 not in d._unchecked_chunks
assert len(d._unchecked_chunks) == 0
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
data = b'0' * opts.chunk_size_bytes
offsets, _ = d.next_offsets()
d.write_unchecked_hmac_data(offsets, data)
ucc = d._unchecked_chunks[offsets.chunk_num]
offsets1, _ = d.next_offsets()
d.write_unchecked_hmac_data(offsets1, data)
ucc1 = d._unchecked_chunks[offsets1.chunk_num]
ucc['decrypted'] = True
ucc1['decrypted'] = True
d.perform_chunked_integrity_check()
assert ucc['ucc'].file_path != d.final_path
assert ucc1['ucc'].file_path != d.final_path
assert d._next_integrity_chunk == 2
assert 0 not in d._unchecked_chunks
assert 1 not in d._unchecked_chunks
assert len(d._unchecked_chunks) == 0
# check integrity with resume
resumefile = pathlib.Path(str(tmpdir.join('resume')))
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
data = b'0' * opts.chunk_size_bytes
md5 = util.new_md5_hasher()
md5.update(data)
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._name = 'blob'
ase._client = mock.MagicMock()
ase._md5 = md5.hexdigest()
rmgr = rops.DownloadResumeManager(resumefile)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
offsets, _ = d.next_offsets()
d.write_unchecked_data(offsets, data)
d.perform_chunked_integrity_check()
assert d._next_integrity_chunk == 1
assert len(d._unchecked_chunks) == 0
dr = rmgr.get_record(ase)
assert dr.next_integrity_chunk == 1
assert dr.md5hexdigest == md5.hexdigest()
def test_finalize_integrity_and_file(tmpdir):
# already finalized
lp = pathlib.Path(str(tmpdir.join('af')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
data = b'0' * ase._size
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
d._finalized = True
d.finalize_integrity()
d.finalize_file()
assert d.final_path.exists()
assert d.final_path.stat().st_size == ase._size
d.final_path.unlink()
# hmac check success
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
signkey = os.urandom(32)
ase._encryption.initialize_hmac = mock.MagicMock()
ase._encryption.initialize_hmac.return_value = hmac.new(
signkey, digestmod=hashlib.sha256)
data = b'0' * (ase._size - 16)
_hmac = hmac.new(signkey, digestmod=hashlib.sha256)
_hmac.update(data)
ase._encryption.encryption_authentication.\
message_authentication_code = util.base64_encode_as_string(
_hmac.digest())
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
d.hmac.update(data)
d.finalize_integrity()
d.finalize_file()
assert d.final_path.exists()
assert d.final_path.stat().st_size == len(data)
# md5 check success
lp = pathlib.Path(str(tmpdir.join('b')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
data = b'0' * ase._size
md5 = util.new_md5_hasher()
md5.update(data)
ase._md5 = util.base64_encode_as_string(md5.digest())
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
d.md5.update(data)
d.finalize_integrity()
d.finalize_file()
assert d.final_path.exists()
assert d.final_path.stat().st_size == len(data)
# no check
lp = pathlib.Path(str(tmpdir.join('c')))
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
data = b'0' * ase._size
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
d.finalize_integrity()
d.finalize_file()
assert d.final_path.exists()
assert d.final_path.stat().st_size == len(data)
# md5 mismatch
lp = pathlib.Path(str(tmpdir.join('d')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
data = b'0' * ase._size
ase._md5 = 'oops'
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
d._allocate_disk_space()
d.md5.update(data)
d.finalize_integrity()
d.finalize_file()
assert not d.final_path.exists()
def test_downloaddescriptor_resume(tmpdir):
resumefile = pathlib.Path(str(tmpdir.join('resume')))
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 256
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
# test no record
rmgr = rops.DownloadResumeManager(resumefile)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# test length mismatch
rmgr.add_or_update_record(str(fp), ase, 0, 0, False, None)
ase._size = 127
rb = d._resume()
assert rb is None
ase._size = 128
# test nothing to resume
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
rmgr.add_or_update_record(str(fp), ase, 0, 0, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# test completion
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
rmgr.add_or_update_record(str(fp), ase, 32, 1, True, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
fp.touch()
rb = d._resume()
assert rb == ase._size
# test encrypted no resume
fp.unlink()
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# test up to chunk
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb == 32
# ensure hmac not populated
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
fp.touch()
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
d.hmac = True
with pytest.raises(RuntimeError):
d._resume()
# md5 hash check
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
data = os.urandom(32)
with fp.open('wb') as f:
f.write(data)
md5 = util.new_md5_hasher()
md5.update(data)
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, md5.hexdigest())
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb == 32
# md5 hash mismatch
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, 'abc')
ase._md5 = 'abc'
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# md5 hash check as page file
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
ase._mode = azmodels.StorageModes.Page
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, md5.hexdigest())
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb == 32