def test_update_resume_for_completed(tmpdir):
resumefile = pathlib.Path(str(tmpdir.join('resume')))
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._name = 'blob'
ase._client = mock.MagicMock()
rmgr = rops.DownloadResumeManager(resumefile)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
offsets, _ = d.next_offsets()
d._update_resume_for_completed()
dr = rmgr.get_record(ase)
assert dr.completed
def test_downloaddescriptor_resume(tmpdir):
resumefile = pathlib.Path(str(tmpdir.join('resume')))
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 256
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
# test no record
rmgr = rops.DownloadResumeManager(resumefile)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# test length mismatch
rmgr.add_or_update_record(str(fp), ase, 0, 0, False, None)
ase._size = 127
rb = d._resume()
assert rb is None
ase._size = 128
# test nothing to resume
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
rmgr.add_or_update_record(str(fp), ase, 0, 0, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# test completion
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
rmgr.add_or_update_record(str(fp), ase, 32, 1, True, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
fp.touch()
rb = d._resume()
assert rb == ase._size
# test encrypted no resume
fp.unlink()
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# test up to chunk
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb == 32
# ensure hmac not populated
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
fp.touch()
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
d.hmac = True
with pytest.raises(RuntimeError):
d._resume()
# md5 hash check
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
data = os.urandom(32)
with fp.open('wb') as f:
f.write(data)
md5 = util.new_md5_hasher()
md5.update(data)
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, md5.hexdigest())
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb == 32
# md5 hash mismatch
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, 'abc')
ase._md5 = 'abc'
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb is None
# md5 hash check as page file
rmgr.delete()
rmgr = rops.DownloadResumeManager(resumefile)
ase = azmodels.StorageEntity('cont')
ase._size = 128
ase._name = 'blob'
ase._client = mock.MagicMock()
ase._mode = azmodels.StorageModes.Page
rmgr.add_or_update_record(str(fp), ase, 32, 1, False, md5.hexdigest())
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
rb = d._resume()
assert rb == 32
def test_perform_chunked_integrity_check(tmpdir):
lp = pathlib.Path(str(tmpdir.join('a')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
offsets, _ = d.next_offsets()
data = b'0' * opts.chunk_size_bytes
d.write_unchecked_data(offsets, data)
d.perform_chunked_integrity_check()
assert d._next_integrity_chunk == 1
assert 0 not in d._unchecked_chunks
assert len(d._unchecked_chunks) == 0
opts = mock.MagicMock()
opts.check_file_md5 = False
opts.chunk_size_bytes = 16
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = b'123'
d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None)
data = b'0' * opts.chunk_size_bytes
offsets, _ = d.next_offsets()
d.write_unchecked_hmac_data(offsets, data)
ucc = d._unchecked_chunks[offsets.chunk_num]
offsets1, _ = d.next_offsets()
d.write_unchecked_hmac_data(offsets1, data)
ucc1 = d._unchecked_chunks[offsets1.chunk_num]
ucc['decrypted'] = True
ucc1['decrypted'] = True
d.perform_chunked_integrity_check()
assert ucc['ucc'].file_path != d.final_path
assert ucc1['ucc'].file_path != d.final_path
assert d._next_integrity_chunk == 2
assert 0 not in d._unchecked_chunks
assert 1 not in d._unchecked_chunks
assert len(d._unchecked_chunks) == 0
# check integrity with resume
resumefile = pathlib.Path(str(tmpdir.join('resume')))
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()
opts.check_file_md5 = True
opts.chunk_size_bytes = 16
data = b'0' * opts.chunk_size_bytes
md5 = util.new_md5_hasher()
md5.update(data)
ase = azmodels.StorageEntity('cont')
ase._size = 32
ase._name = 'blob'
ase._client = mock.MagicMock()
ase._md5 = md5.hexdigest()
rmgr = rops.DownloadResumeManager(resumefile)
d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr)
offsets, _ = d.next_offsets()
d.write_unchecked_data(offsets, data)
d.perform_chunked_integrity_check()
assert d._next_integrity_chunk == 1
assert len(d._unchecked_chunks) == 0
dr = rmgr.get_record(ase)
assert dr.next_integrity_chunk == 1
assert dr.md5hexdigest == md5.hexdigest()
def test_download_resume_manager(tmpdir):
tmpdb = pathlib.Path(str(tmpdir.join('tmp.db')))
tmpdb_dat = pathlib.Path(str(tmpdir.join('tmp.db.dat')))
drm = ops.DownloadResumeManager(tmpdb)
assert drm._data is not None
drm.close()
assert drm._data is None
if not util.on_python2() and util.on_windows():
assert tmpdb_dat.exists()
else:
assert tmpdb.exists()
drm.delete()
assert drm._data is None
if not util.on_python2() and util.on_windows():
assert not tmpdb_dat.exists()
else:
assert not tmpdb.exists()
ase = mock.MagicMock()
ase._name = 'name'
ase._client.primary_endpoint = 'ep'
ase._size = 16
final_path = 'fp'
drm = ops.DownloadResumeManager(tmpdb)
drm.add_or_update_record(final_path, ase, 2, 0, False, None)
d = drm.get_record(ase)
assert d.final_path == final_path
drm.add_or_update_record(final_path, ase, 2, 1, False, 'abc')
d = drm.get_record(ase)
assert d.final_path == final_path
assert not d.completed
assert d.next_integrity_chunk == 1
assert d.md5hexdigest == 'abc'
drm.add_or_update_record(final_path, ase, 2, 1, True, None)
d = drm.get_record(ase)
assert d.final_path == final_path
assert d.completed
assert d.next_integrity_chunk == 1
assert d.md5hexdigest == 'abc'
# idempotent check after completed
drm.add_or_update_record(final_path, ase, 2, 1, True, None)
d = drm.get_record(ase)
assert d.final_path == final_path
assert d.completed
assert d.next_integrity_chunk == 1
assert d.md5hexdigest == 'abc'
drm.close()
assert drm._data is None
if not util.on_python2() and util.on_windows():
assert tmpdb_dat.exists()
else:
assert tmpdb.exists()
drm.delete()
assert drm._data is None
if not util.on_python2() and util.on_windows():
assert not tmpdb_dat.exists()
else:
assert not tmpdb.exists()
# oserror path
with mock.patch('blobxfer.util.on_windows', return_value=False):
drm.delete()
assert drm._data is None
# oserror path
with mock.patch('blobxfer.util.on_windows', return_value=True):
with mock.patch('blobxfer.util.on_python2', return_value=False):
drm.delete()
assert drm._data is None
