def test_perform_chunked_integrity_check(tmpdir): lp = pathlib.Path(str(tmpdir.join('a'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) offsets, _ = d.next_offsets() data = b'0' * opts.chunk_size_bytes d.write_unchecked_data(offsets, data) d.perform_chunked_integrity_check() assert d._next_integrity_chunk == 1 assert 0 not in d._unchecked_chunks assert len(d._unchecked_chunks) == 0 opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = b'123' d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) data = b'0' * opts.chunk_size_bytes offsets, _ = d.next_offsets() d.write_unchecked_hmac_data(offsets, data) ucc = d._unchecked_chunks[offsets.chunk_num] offsets1, _ = d.next_offsets() d.write_unchecked_hmac_data(offsets1, data) ucc1 = d._unchecked_chunks[offsets1.chunk_num] ucc['decrypted'] = True ucc1['decrypted'] = True d.perform_chunked_integrity_check() assert ucc['ucc'].file_path != d.final_path assert ucc1['ucc'].file_path != d.final_path assert d._next_integrity_chunk == 2 assert 0 not in d._unchecked_chunks assert 1 not in d._unchecked_chunks assert len(d._unchecked_chunks) == 0 # check integrity with resume resumefile = pathlib.Path(str(tmpdir.join('resume'))) fp = pathlib.Path(str(tmpdir.join('fp'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 data = b'0' * opts.chunk_size_bytes md5 = util.new_md5_hasher() md5.update(data) ase = azmodels.StorageEntity('cont') ase._size = 32 ase._name = 'blob' ase._client = mock.MagicMock() ase._md5 = md5.hexdigest() rmgr = rops.DownloadResumeManager(resumefile) d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) offsets, _ = d.next_offsets() d.write_unchecked_data(offsets, data) d.perform_chunked_integrity_check() assert d._next_integrity_chunk == 1 assert len(d._unchecked_chunks) == 0 dr = rmgr.get_record(ase) assert dr.next_integrity_chunk == 1 assert dr.md5hexdigest == md5.hexdigest()
def test_finalize_integrity_and_file(tmpdir): # already finalized lp = pathlib.Path(str(tmpdir.join('af'))) opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 data = b'0' * ase._size d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._allocate_disk_space() d._finalized = True d.finalize_integrity() d.finalize_file() assert d.final_path.exists() assert d.final_path.stat().st_size == ase._size d.final_path.unlink() # hmac check success lp = pathlib.Path(str(tmpdir.join('a'))) opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = b'123' signkey = os.urandom(32) ase._encryption.initialize_hmac = mock.MagicMock() ase._encryption.initialize_hmac.return_value = hmac.new( signkey, digestmod=hashlib.sha256) data = b'0' * (ase._size - 16) _hmac = hmac.new(signkey, digestmod=hashlib.sha256) _hmac.update(data) ase._encryption.encryption_authentication.\ message_authentication_code = util.base64_encode_as_string( _hmac.digest()) d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._allocate_disk_space() d.hmac.update(data) d.finalize_integrity() d.finalize_file() assert d.final_path.exists() assert d.final_path.stat().st_size == len(data) # md5 check success lp = pathlib.Path(str(tmpdir.join('b'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 data = b'0' * ase._size md5 = util.new_md5_hasher() md5.update(data) ase._md5 = util.base64_encode_as_string(md5.digest()) d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._allocate_disk_space() d.md5.update(data) d.finalize_integrity() d.finalize_file() assert d.final_path.exists() assert d.final_path.stat().st_size == len(data) # no check lp = pathlib.Path(str(tmpdir.join('c'))) opts = mock.MagicMock() opts.check_file_md5 = False opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 data = b'0' * ase._size d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._allocate_disk_space() d.finalize_integrity() d.finalize_file() assert d.final_path.exists() assert d.final_path.stat().st_size == len(data) # md5 mismatch lp = pathlib.Path(str(tmpdir.join('d'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 ase = azmodels.StorageEntity('cont') ase._size = 32 data = b'0' * ase._size ase._md5 = 'oops' d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._allocate_disk_space() d.md5.update(data) d.finalize_integrity() d.finalize_file() assert not d.final_path.exists()
def test_downloaddescriptor_resume(tmpdir): resumefile = pathlib.Path(str(tmpdir.join('resume'))) fp = pathlib.Path(str(tmpdir.join('fp'))) opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 256 ase = azmodels.StorageEntity('cont') ase._size = 128 ase._name = 'blob' ase._client = mock.MagicMock() # test no record rmgr = rops.DownloadResumeManager(resumefile) d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) rb = d._resume() assert rb is None # test length mismatch rmgr.add_or_update_record(str(fp), ase, 0, 0, False, None) ase._size = 127 rb = d._resume() assert rb is None ase._size = 128 # test nothing to resume rmgr.delete() rmgr = rops.DownloadResumeManager(resumefile) rmgr.add_or_update_record(str(fp), ase, 0, 0, False, None) d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) rb = d._resume() assert rb is None # test completion rmgr.delete() rmgr = rops.DownloadResumeManager(resumefile) rmgr.add_or_update_record(str(fp), ase, 32, 1, True, None) d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) fp.touch() rb = d._resume() assert rb == ase._size # test encrypted no resume fp.unlink() rmgr.delete() rmgr = rops.DownloadResumeManager(resumefile) ase._encryption = mock.MagicMock() ase._encryption.symmetric_key = b'123' rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None) d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) rb = d._resume() assert rb is None # test up to chunk rmgr.delete() rmgr = rops.DownloadResumeManager(resumefile) ase = azmodels.StorageEntity('cont') ase._size = 128 ase._name = 'blob' ase._client = mock.MagicMock() rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None) d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) rb = d._resume() assert rb == 32 # ensure hmac not populated rmgr.delete() rmgr = rops.DownloadResumeManager(resumefile) ase = azmodels.StorageEntity('cont') ase._size = 128 ase._name = 'blob' ase._client = mock.MagicMock() fp.touch() rmgr.add_or_update_record(str(fp), ase, 32, 1, False, None) d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) d.hmac = True with pytest.raises(RuntimeError): d._resume() # md5 hash check rmgr.delete() rmgr = rops.DownloadResumeManager(resumefile) data = os.urandom(32) with fp.open('wb') as f: f.write(data) md5 = util.new_md5_hasher() md5.update(data) rmgr.add_or_update_record(str(fp), ase, 32, 1, False, md5.hexdigest()) d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) rb = d._resume() assert rb == 32 # md5 hash mismatch rmgr.delete() rmgr = rops.DownloadResumeManager(resumefile) rmgr.add_or_update_record(str(fp), ase, 32, 1, False, 'abc') ase._md5 = 'abc' d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) rb = d._resume() assert rb is None # md5 hash check as page file rmgr.delete() rmgr = rops.DownloadResumeManager(resumefile) ase = azmodels.StorageEntity('cont') ase._size = 128 ase._name = 'blob' ase._client = mock.MagicMock() ase._mode = azmodels.StorageModes.Page rmgr.add_or_update_record(str(fp), ase, 32, 1, False, md5.hexdigest()) d = models.Descriptor(fp, ase, opts, mock.MagicMock(), rmgr) rb = d._resume() assert rb == 32