def test_bitmap_does_not_exist(nbd_env): vol1 = create_volume(nbd_env, "raw", "sparse") vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1) bitmap = str(uuid.uuid4()) # Add another bitmap to the volume. qemuimg.bitmap_add(vol2.volumePath, str(uuid.uuid4())).run() config = { "sd_id": vol2.sdUUID, "img_id": vol2.imgUUID, "vol_id": vol2.volUUID, "readonly": True, "bitmap": bitmap, } # Starting server will fail since bitmap does not exist. with pytest.raises(se.BitmapDoesNotExist): with nbd_server(config): pass
def test_bitmap_disabled(nbd_env): vol1 = create_volume(nbd_env, "qcow2", "sparse") vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1) bitmap = str(uuid.uuid4()) # Add bitmap to both volumes, but disable it on vol1. qemuimg.bitmap_add(vol1.volumePath, bitmap, enable=False).run() qemuimg.bitmap_add(vol2.volumePath, bitmap).run() config = { "sd_id": vol2.sdUUID, "img_id": vol2.imgUUID, "vol_id": vol2.volUUID, "readonly": True, "bitmap": bitmap, } # Starting server will fail since the bitmap in vol1 is disabled. with pytest.raises(se.InvalidBitmapChain): with nbd_server(config): pass
def test_add_only_valid_bitmaps(vol_chain): bitmap = 'bitmap' # Add new bitmap to base volume op = qemuimg.bitmap_add(vol_chain.base_vol, bitmap) op.run() # Add invalid bitmap to base volume op = qemuimg.bitmap_add(vol_chain.base_vol, 'disabled', enable=False) op.run() # Add bitmaps from base volume to top volume bitmaps.add_bitmaps(vol_chain.base_vol, vol_chain.top_vol) info = qemuimg.info(vol_chain.top_vol) assert info['format-specific']['data']['bitmaps'] == [ { "flags": ["auto"], "name": bitmap, "granularity": 65536 }, ]
def test_merge_bitmaps_failed_to_add_bitmap(monkeypatch, vol_chain): bitmap = 'bitmap' # Add new bitmap to top volume op = qemuimg.bitmap_add(vol_chain.top_vol, bitmap) op.run() monkeypatch.setattr(qemuimg, "bitmap_add", qemuimg_failure) with pytest.raises(exception.AddBitmapError): bitmaps.merge_bitmaps(vol_chain.base_vol, vol_chain.top_vol) info = qemuimg.info(vol_chain.base_vol) assert 'bitmaps' not in info['format-specific']['data']
def test_bitmap_in_use_single_volume(nbd_env): vol = create_volume(nbd_env, "qcow2", "sparse") bitmap = str(uuid.uuid4()) qemuimg.bitmap_add(vol.volumePath, bitmap).run() # Simulate qemu crash, leaving bitmaps with the "in-use" flag by opening # the image for writing and killing the process. qemuio.abort(vol.volumePath) config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, "readonly": True, "bitmap": bitmap, } # Starting server will fail since the bitmap in vol is in use. with pytest.raises(se.InvalidBitmapChain): with nbd_server(config): pass
def test_volume_chain_copy_with_bitmaps(user_mount, fake_scheduler, env_type, sd_version, copy_seq): bitmaps = ['bitmap1', 'bitmap2'] data_center = os.path.join(user_mount.path, "data-center") with make_env(env_type, sc.COW_FORMAT, sc.COW_FORMAT, chain_length=len(copy_seq), sd_version=sd_version, src_qcow2_compat='1.1', data_center=data_center) as env: for index in copy_seq: # Add bitmaps to src volume vol_path = env.src_chain[index].getVolumePath() for bitmap in bitmaps: op = qemuimg.bitmap_add(vol_path, bitmap) op.run() job_id = make_uuid() src_vol = env.src_chain[index] dst_vol = env.dst_chain[index] source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID) job = copy_data.Job(job_id, 0, source, dest, copy_bitmaps=True) job.run() for index in copy_seq: dst_vol = env.dst_chain[index] info = qemuimg.info(dst_vol.getVolumePath()) assert info["format-specific"]["data"]["bitmaps"] == [ { "flags": ["auto"], "name": bitmaps[0], "granularity": 65536 }, { "flags": ["auto"], "name": bitmaps[1], "granularity": 65536 }, ]
def _add_bitmap(vol_path, bitmap, granularity, enable=True): log.info("Add bitmap %s to %r", bitmap, vol_path) try: op = qemuimg.bitmap_add(vol_path, bitmap, enable=enable, granularity=granularity) op.run() except cmdutils.Error as e: raise exception.AddBitmapError( reason="Failed to add bitmap: {}".format(e), bitmap=bitmap, dst_vol_path=vol_path)
def test_clear_bitmaps(tmp_mount, vol_chain): bitmap_1 = 'bitmap_1' bitmap_2 = 'bitmap_2' # Add new bitmaps to top volume for bitmap in [bitmap_1, bitmap_2]: op = qemuimg.bitmap_add(vol_chain.top_vol, bitmap) op.run() # Clear top volume bitmaps bitmaps.clear_bitmaps(vol_chain.top_vol) info = qemuimg.info(vol_chain.top_vol) vol_bitmaps = info["format-specific"]["data"].get("bitmaps", []) assert not vol_bitmaps
def test_bitmap_in_use(nbd_env): vol1 = create_volume(nbd_env, "qcow2", "sparse") vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1) bitmap = str(uuid.uuid4()) qemuimg.bitmap_add(vol1.volumePath, bitmap).run() qemuimg.bitmap_add(vol2.volumePath, bitmap).run() # Simulate qemu crash, leaving bitmaps with the "in-use" flag by opening # the image for writing and killing the process. subprocess.run(["qemu-io", "-c", "sigraise 9", vol2.volumePath]) config = { "sd_id": vol2.sdUUID, "img_id": vol2.imgUUID, "vol_id": vol2.volUUID, "readonly": True, "bitmap": bitmap, } # Starting server will fail since the bitmap in vol2 is in in use. with pytest.raises(se.InvalidBitmapChain): with nbd_server(config): pass
def test_merge_bitmaps_failed(monkeypatch, vol_chain): bitmap = 'bitmap' # Add new bitmap to top volume op = qemuimg.bitmap_add(vol_chain.top_vol, bitmap) op.run() monkeypatch.setattr(qemuimg, "bitmap_merge", qemuimg_failure) with pytest.raises(exception.MergeBitmapError): bitmaps.merge_bitmaps(vol_chain.base_vol, vol_chain.top_vol) info = qemuimg.info(vol_chain.base_vol) # TODO: test that this bitmap is empty assert info['bitmaps'] == [ { "flags": ['auto'], "name": bitmap, "granularity": 65536 }, ]
def test_bitmap_already_exists(fake_scheduler, env_type): job_id = make_uuid() bitmap = "bitmap" with make_env(env_type, sc.name2type('cow')) as env: env_vol = env.chain[0] op = qemuimg.bitmap_add(env_vol.getVolumePath(), bitmap) op.run() generation = env_vol.getMetaParam(sc.GENERATION) vol = dict(endpoint_type='div', sd_id=env_vol.sdUUID, img_id=env_vol.imgUUID, vol_id=env_vol.volUUID, generation=generation) job = add_bitmap.Job(job_id, 0, vol, bitmap) job.run() assert job.status == jobs.STATUS.FAILED assert type(job.error) == se.GeneralException assert env_vol.getLegality() == sc.LEGAL_VOL assert env_vol.getMetaParam(sc.GENERATION) == generation
def test_clear_bitmaps(fake_scheduler, env_type): with make_env(env_type, sc.name2type('cow')) as env: top_vol = env.chain[0] # Add new bitmaps to top volume for bitmap in ['bitmap_1', 'bitmap_2']: op = qemuimg.bitmap_add(top_vol.getVolumePath(), bitmap) op.run() # Clear the created bitmap generation = top_vol.getMetaParam(sc.GENERATION) vol = { 'endpoint_type': 'div', 'sd_id': top_vol.sdUUID, 'img_id': top_vol.imgUUID, 'vol_id': top_vol.volUUID, 'generation': generation } job = clear_bitmaps.Job(make_uuid(), 0, vol) job.run() assert jobs.STATUS.DONE == job.status vol_info = qemuimg.info(top_vol.getVolumePath()) assert "bitmaps" not in vol_info["format-specific"]["data"] assert top_vol.getMetaParam(sc.GENERATION) == generation + 1
def test_bitmap_backing_chain(nbd_env): vol1 = create_volume(nbd_env, "raw", "sparse") # Write first cluster to vol1 - this cluster is not recorded in any bitmap. qemuio.write_pattern(vol1.volumePath, "raw", offset=1 * MiB, len=64 * KiB, pattern=0xf1) # Simulate a snapshot - bitmap1 is created empty in vol2. vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1) bitmap1 = str(uuid.uuid4()) qemuimg.bitmap_add(vol2.volumePath, bitmap1).run() # Write second cluster in vol2 - this cluster is recorded only in bitmap 1. qemuio.write_pattern(vol2.volumePath, "qcow2", offset=2 * MiB, len=64 * KiB, pattern=0xf2) # Simulate another snapshot - bitmap1 is created empty in vol3. vol3 = create_volume(nbd_env, "qcow2", "sparse", parent=vol2) qemuimg.bitmap_add(vol3.volumePath, bitmap1).run() # Simulate backup - bitmap 2 is created in vol3. bitmap2 = str(uuid.uuid4()) qemuimg.bitmap_add(vol3.volumePath, bitmap2).run() # Write third cluster in vol3. This cluster is recorded in both bitmaps. qemuio.write_pattern(vol3.volumePath, "qcow2", offset=3 * MiB, len=64 * KiB, pattern=0xf3) # Test bitmap 1 - recording changes since bitmap 1 was added. config = { "sd_id": vol3.sdUUID, "img_id": vol3.imgUUID, "vol_id": vol3.volUUID, "readonly": True, "bitmap": bitmap1, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) dirty_extents = [(e.length, e.dirty) for e in extents[c.dirty_bitmap]] assert dirty_extents == [ (2 * MiB, False), (64 * KiB, True), (1 * MiB - 64 * KiB, False), (64 * KiB, True), (nbd_env.virtual_size - 3 * MiB - 64 * KiB, False), ] assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB # Test bitmap 2 - recording changes since bitmap 2 was added. config = { "sd_id": vol3.sdUUID, "img_id": vol3.imgUUID, "vol_id": vol3.volUUID, "readonly": True, "bitmap": bitmap2, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) dirty_extents = [(e.length, e.dirty) for e in extents[c.dirty_bitmap]] assert dirty_extents == [ (3 * MiB, False), (64 * KiB, True), (nbd_env.virtual_size - 3 * MiB - 64 * KiB, False), ] assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
def test_bitmap_single_volume(nbd_env): vol = create_volume(nbd_env, "qcow2", "sparse") # Write first cluster - this cluster is not recorded in any bitmap. qemuio.write_pattern(vol.volumePath, "qcow2", offset=1 * MiB, len=64 * KiB, pattern=0xf1) # Add bitmap 1 and write second cluster. bitmap1 = str(uuid.uuid4()) qemuimg.bitmap_add(vol.volumePath, bitmap1).run() qemuio.write_pattern(vol.volumePath, "qcow2", offset=2 * MiB, len=64 * KiB, pattern=0xf2) # Add bitmap 2 and write third cluster. bitmap2 = str(uuid.uuid4()) qemuimg.bitmap_add(vol.volumePath, bitmap2).run() qemuio.write_pattern(vol.volumePath, "qcow2", offset=3 * MiB, len=64 * KiB, pattern=0xf3) # Test bitmap 1 - recording changes since bitmap 1 was added. config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, "readonly": True, "bitmap": bitmap1, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) dirty_extents = [(e.length, e.dirty) for e in extents[c.dirty_bitmap]] assert dirty_extents == [ (2 * MiB, False), (64 * KiB, True), (1 * MiB - 64 * KiB, False), (64 * KiB, True), (nbd_env.virtual_size - 3 * MiB - 64 * KiB, False), ] assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB # Test bitmap 2 - recording changes since bitmap 2 was added. config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, "readonly": True, "bitmap": bitmap2, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) dirty_extents = [(e.length, e.dirty) for e in extents[c.dirty_bitmap]] assert dirty_extents == [ (3 * MiB, False), (64 * KiB, True), (nbd_env.virtual_size - 3 * MiB - 64 * KiB, False), ] assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
def test_create_with_bitmaps(tmp_storage, tmp_repo, fake_access, fake_rescan, tmp_db, fake_task, fake_sanlock, domain_version): sd_uuid = str(uuid.uuid4()) dev = tmp_storage.create_device(20 * GiB) lvm.createVG(sd_uuid, [dev], blockSD.STORAGE_UNREADY_DOMAIN_TAG, 128) vg = lvm.getVG(sd_uuid) dom = blockSD.BlockStorageDomain.create(sdUUID=sd_uuid, domainName="domain", domClass=sd.DATA_DOMAIN, vgUUID=vg.uuid, version=domain_version, storageType=sd.ISCSI_DOMAIN) sdCache.knownSDs[sd_uuid] = blockSD.findDomain sdCache.manuallyAddDomain(dom) dom.refresh() dom.attach(tmp_repo.pool_id) img_uuid = str(uuid.uuid4()) base_vol_uuid = str(uuid.uuid4()) base_vol_capacity = GiB top_vol_uuid = str(uuid.uuid4()) vol_capacity = 2 * base_vol_capacity bitmap_names = ['bitmap1', 'bitmap2'] # Create base volume. dom.createVolume(imgUUID=img_uuid, capacity=base_vol_capacity, volFormat=sc.COW_FORMAT, preallocate=sc.SPARSE_VOL, diskType='DATA', volUUID=base_vol_uuid, desc="Test base volume", srcImgUUID=sc.BLANK_UUID, srcVolUUID=sc.BLANK_UUID) base_vol = dom.produceVolume(img_uuid, base_vol_uuid) base_vol_path = base_vol.getVolumePath() # Prepare the volume in order to create bitmaps base_vol.prepare() # Add new bitmaps to base volume for bitmap_name in bitmap_names: op = qemuimg.bitmap_add( base_vol_path, bitmap_name, ) op.run() # Teardown the volume, test if prepare() will be # called during the snapshot creation base_vol.teardown(sd_uuid, base_vol_uuid) # Create top volume with bitmaps. dom.createVolume(imgUUID=img_uuid, capacity=vol_capacity, volFormat=sc.COW_FORMAT, preallocate=sc.SPARSE_VOL, diskType='DATA', volUUID=top_vol_uuid, desc="Test top volume", srcImgUUID=base_vol.imgUUID, srcVolUUID=base_vol.volUUID, add_bitmaps=True) top_vol = dom.produceVolume(img_uuid, top_vol_uuid) top_vol_path = top_vol.getVolumePath() # Prepare the volume in order to get # info on the bitmaps top_vol.prepare() info = qemuimg.info(top_vol_path) # Teardown top volume base_vol.teardown(sd_uuid, top_vol_uuid) assert info['format-specific']['data']['bitmaps'] == [ { "flags": ["auto"], "name": bitmap_names[0], "granularity": 65536 }, { "flags": ["auto"], "name": bitmap_names[1], "granularity": 65536 }, ]
def test_create_volume_with_bitmaps(user_domain, local_fallocate): if user_domain.getVersion() == 3: pytest.skip("Bitmaps operations not supported in v3 domains") parent_img_uuid = str(uuid.uuid4()) parent_vol_uuid = str(uuid.uuid4()) bitmap_names = ['bitmap1', 'bitmap2'] # Create base volume user_domain.createVolume( imgUUID=parent_img_uuid, capacity=SPARSE_VOL_SIZE, volFormat=sc.COW_FORMAT, preallocate=sc.SPARSE_VOL, diskType="DATA", volUUID=parent_vol_uuid, desc="test", srcImgUUID=sc.BLANK_UUID, srcVolUUID=sc.BLANK_UUID) parent_vol = user_domain.produceVolume( parent_img_uuid, parent_vol_uuid) parent_vol_path = parent_vol.getVolumePath() # Add new bitmaps to base volume for bitmap_name in bitmap_names: op = qemuimg.bitmap_add( parent_vol_path, bitmap_name, ) op.run() vol_uuid = str(uuid.uuid4()) # Create top volume user_domain.createVolume( imgUUID=parent_img_uuid, capacity=SPARSE_VOL_SIZE, volFormat=sc.COW_FORMAT, preallocate=sc.SPARSE_VOL, diskType='DATA', volUUID=vol_uuid, desc="Test volume", srcImgUUID=parent_vol.imgUUID, srcVolUUID=parent_vol.volUUID, add_bitmaps=True ) vol = user_domain.produceVolume( parent_img_uuid, vol_uuid) vol_path = vol.getVolumePath() info = qemuimg.info(vol_path) assert info['format-specific']['data']['bitmaps'] == [ { "flags": ["auto"], "name": bitmap_names[0], "granularity": 65536 }, { "flags": ["auto"], "name": bitmap_names[1], "granularity": 65536 }, ]