def test_bitmap_does_not_exist(nbd_env):
vol1 = create_volume(nbd_env, "raw", "sparse")
vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1)
bitmap = str(uuid.uuid4())
# Add another bitmap to the volume.
qemuimg.bitmap_add(vol2.volumePath, str(uuid.uuid4())).run()
config = {
"sd_id": vol2.sdUUID,
"img_id": vol2.imgUUID,
"vol_id": vol2.volUUID,
"readonly": True,
"bitmap": bitmap,
}
# Starting server will fail since bitmap does not exist.
with pytest.raises(se.BitmapDoesNotExist):
with nbd_server(config):
pass
def test_bitmap_disabled(nbd_env):
vol1 = create_volume(nbd_env, "qcow2", "sparse")
vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1)
bitmap = str(uuid.uuid4())
# Add bitmap to both volumes, but disable it on vol1.
qemuimg.bitmap_add(vol1.volumePath, bitmap, enable=False).run()
qemuimg.bitmap_add(vol2.volumePath, bitmap).run()
config = {
"sd_id": vol2.sdUUID,
"img_id": vol2.imgUUID,
"vol_id": vol2.volUUID,
"readonly": True,
"bitmap": bitmap,
}
# Starting server will fail since the bitmap in vol1 is disabled.
with pytest.raises(se.InvalidBitmapChain):
with nbd_server(config):
pass
def test_add_only_valid_bitmaps(vol_chain):
bitmap = 'bitmap'
# Add new bitmap to base volume
op = qemuimg.bitmap_add(vol_chain.base_vol, bitmap)
op.run()
# Add invalid bitmap to base volume
op = qemuimg.bitmap_add(vol_chain.base_vol, 'disabled', enable=False)
op.run()
# Add bitmaps from base volume to top volume
bitmaps.add_bitmaps(vol_chain.base_vol, vol_chain.top_vol)
info = qemuimg.info(vol_chain.top_vol)
assert info['format-specific']['data']['bitmaps'] == [
{
"flags": ["auto"],
"name": bitmap,
"granularity": 65536
},
]
def test_merge_bitmaps_failed_to_add_bitmap(monkeypatch, vol_chain):
bitmap = 'bitmap'
# Add new bitmap to top volume
op = qemuimg.bitmap_add(vol_chain.top_vol, bitmap)
op.run()
monkeypatch.setattr(qemuimg, "bitmap_add", qemuimg_failure)
with pytest.raises(exception.AddBitmapError):
bitmaps.merge_bitmaps(vol_chain.base_vol, vol_chain.top_vol)
info = qemuimg.info(vol_chain.base_vol)
assert 'bitmaps' not in info['format-specific']['data']
def test_bitmap_in_use_single_volume(nbd_env):
vol = create_volume(nbd_env, "qcow2", "sparse")
bitmap = str(uuid.uuid4())
qemuimg.bitmap_add(vol.volumePath, bitmap).run()
# Simulate qemu crash, leaving bitmaps with the "in-use" flag by opening
# the image for writing and killing the process.
qemuio.abort(vol.volumePath)
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
"readonly": True,
"bitmap": bitmap,
}
# Starting server will fail since the bitmap in vol is in use.
with pytest.raises(se.InvalidBitmapChain):
with nbd_server(config):
pass
def test_volume_chain_copy_with_bitmaps(user_mount, fake_scheduler, env_type,
sd_version, copy_seq):
bitmaps = ['bitmap1', 'bitmap2']
data_center = os.path.join(user_mount.path, "data-center")
with make_env(env_type,
sc.COW_FORMAT,
sc.COW_FORMAT,
chain_length=len(copy_seq),
sd_version=sd_version,
src_qcow2_compat='1.1',
data_center=data_center) as env:
for index in copy_seq:
# Add bitmaps to src volume
vol_path = env.src_chain[index].getVolumePath()
for bitmap in bitmaps:
op = qemuimg.bitmap_add(vol_path, bitmap)
op.run()
job_id = make_uuid()
src_vol = env.src_chain[index]
dst_vol = env.dst_chain[index]
source = dict(endpoint_type='div',
sd_id=src_vol.sdUUID,
img_id=src_vol.imgUUID,
vol_id=src_vol.volUUID)
dest = dict(endpoint_type='div',
sd_id=dst_vol.sdUUID,
img_id=dst_vol.imgUUID,
vol_id=dst_vol.volUUID)
job = copy_data.Job(job_id, 0, source, dest, copy_bitmaps=True)
job.run()
for index in copy_seq:
dst_vol = env.dst_chain[index]
info = qemuimg.info(dst_vol.getVolumePath())
assert info["format-specific"]["data"]["bitmaps"] == [
{
"flags": ["auto"],
"name": bitmaps[0],
"granularity": 65536
},
{
"flags": ["auto"],
"name": bitmaps[1],
"granularity": 65536
},
]
def _add_bitmap(vol_path, bitmap, granularity, enable=True):
log.info("Add bitmap %s to %r", bitmap, vol_path)
try:
op = qemuimg.bitmap_add(vol_path,
bitmap,
enable=enable,
granularity=granularity)
op.run()
except cmdutils.Error as e:
raise exception.AddBitmapError(
reason="Failed to add bitmap: {}".format(e),
bitmap=bitmap,
dst_vol_path=vol_path)
def test_clear_bitmaps(tmp_mount, vol_chain):
bitmap_1 = 'bitmap_1'
bitmap_2 = 'bitmap_2'
# Add new bitmaps to top volume
for bitmap in [bitmap_1, bitmap_2]:
op = qemuimg.bitmap_add(vol_chain.top_vol, bitmap)
op.run()
# Clear top volume bitmaps
bitmaps.clear_bitmaps(vol_chain.top_vol)
info = qemuimg.info(vol_chain.top_vol)
vol_bitmaps = info["format-specific"]["data"].get("bitmaps", [])
assert not vol_bitmaps
def test_bitmap_in_use(nbd_env):
vol1 = create_volume(nbd_env, "qcow2", "sparse")
vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1)
bitmap = str(uuid.uuid4())
qemuimg.bitmap_add(vol1.volumePath, bitmap).run()
qemuimg.bitmap_add(vol2.volumePath, bitmap).run()
# Simulate qemu crash, leaving bitmaps with the "in-use" flag by opening
# the image for writing and killing the process.
subprocess.run(["qemu-io", "-c", "sigraise 9", vol2.volumePath])
config = {
"sd_id": vol2.sdUUID,
"img_id": vol2.imgUUID,
"vol_id": vol2.volUUID,
"readonly": True,
"bitmap": bitmap,
}
# Starting server will fail since the bitmap in vol2 is in in use.
with pytest.raises(se.InvalidBitmapChain):
with nbd_server(config):
pass
def test_merge_bitmaps_failed(monkeypatch, vol_chain):
bitmap = 'bitmap'
# Add new bitmap to top volume
op = qemuimg.bitmap_add(vol_chain.top_vol, bitmap)
op.run()
monkeypatch.setattr(qemuimg, "bitmap_merge", qemuimg_failure)
with pytest.raises(exception.MergeBitmapError):
bitmaps.merge_bitmaps(vol_chain.base_vol, vol_chain.top_vol)
info = qemuimg.info(vol_chain.base_vol)
# TODO: test that this bitmap is empty
assert info['bitmaps'] == [
{
"flags": ['auto'],
"name": bitmap,
"granularity": 65536
},
]
def test_bitmap_already_exists(fake_scheduler, env_type):
job_id = make_uuid()
bitmap = "bitmap"
with make_env(env_type, sc.name2type('cow')) as env:
env_vol = env.chain[0]
op = qemuimg.bitmap_add(env_vol.getVolumePath(), bitmap)
op.run()
generation = env_vol.getMetaParam(sc.GENERATION)
vol = dict(endpoint_type='div',
sd_id=env_vol.sdUUID,
img_id=env_vol.imgUUID,
vol_id=env_vol.volUUID,
generation=generation)
job = add_bitmap.Job(job_id, 0, vol, bitmap)
job.run()
assert job.status == jobs.STATUS.FAILED
assert type(job.error) == se.GeneralException
assert env_vol.getLegality() == sc.LEGAL_VOL
assert env_vol.getMetaParam(sc.GENERATION) == generation
def test_clear_bitmaps(fake_scheduler, env_type):
with make_env(env_type, sc.name2type('cow')) as env:
top_vol = env.chain[0]
# Add new bitmaps to top volume
for bitmap in ['bitmap_1', 'bitmap_2']:
op = qemuimg.bitmap_add(top_vol.getVolumePath(), bitmap)
op.run()
# Clear the created bitmap
generation = top_vol.getMetaParam(sc.GENERATION)
vol = {
'endpoint_type': 'div',
'sd_id': top_vol.sdUUID,
'img_id': top_vol.imgUUID,
'vol_id': top_vol.volUUID,
'generation': generation
}
job = clear_bitmaps.Job(make_uuid(), 0, vol)
job.run()
assert jobs.STATUS.DONE == job.status
vol_info = qemuimg.info(top_vol.getVolumePath())
assert "bitmaps" not in vol_info["format-specific"]["data"]
assert top_vol.getMetaParam(sc.GENERATION) == generation + 1
def test_bitmap_backing_chain(nbd_env):
vol1 = create_volume(nbd_env, "raw", "sparse")
# Write first cluster to vol1 - this cluster is not recorded in any bitmap.
qemuio.write_pattern(vol1.volumePath,
"raw",
offset=1 * MiB,
len=64 * KiB,
pattern=0xf1)
# Simulate a snapshot - bitmap1 is created empty in vol2.
vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1)
bitmap1 = str(uuid.uuid4())
qemuimg.bitmap_add(vol2.volumePath, bitmap1).run()
# Write second cluster in vol2 - this cluster is recorded only in bitmap 1.
qemuio.write_pattern(vol2.volumePath,
"qcow2",
offset=2 * MiB,
len=64 * KiB,
pattern=0xf2)
# Simulate another snapshot - bitmap1 is created empty in vol3.
vol3 = create_volume(nbd_env, "qcow2", "sparse", parent=vol2)
qemuimg.bitmap_add(vol3.volumePath, bitmap1).run()
# Simulate backup - bitmap 2 is created in vol3.
bitmap2 = str(uuid.uuid4())
qemuimg.bitmap_add(vol3.volumePath, bitmap2).run()
# Write third cluster in vol3. This cluster is recorded in both bitmaps.
qemuio.write_pattern(vol3.volumePath,
"qcow2",
offset=3 * MiB,
len=64 * KiB,
pattern=0xf3)
# Test bitmap 1 - recording changes since bitmap 1 was added.
config = {
"sd_id": vol3.sdUUID,
"img_id": vol3.imgUUID,
"vol_id": vol3.volUUID,
"readonly": True,
"bitmap": bitmap1,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
dirty_extents = [(e.length, e.dirty)
for e in extents[c.dirty_bitmap]]
assert dirty_extents == [
(2 * MiB, False),
(64 * KiB, True),
(1 * MiB - 64 * KiB, False),
(64 * KiB, True),
(nbd_env.virtual_size - 3 * MiB - 64 * KiB, False),
]
assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
# Test bitmap 2 - recording changes since bitmap 2 was added.
config = {
"sd_id": vol3.sdUUID,
"img_id": vol3.imgUUID,
"vol_id": vol3.volUUID,
"readonly": True,
"bitmap": bitmap2,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
dirty_extents = [(e.length, e.dirty)
for e in extents[c.dirty_bitmap]]
assert dirty_extents == [
(3 * MiB, False),
(64 * KiB, True),
(nbd_env.virtual_size - 3 * MiB - 64 * KiB, False),
]
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
def test_bitmap_single_volume(nbd_env):
vol = create_volume(nbd_env, "qcow2", "sparse")
# Write first cluster - this cluster is not recorded in any bitmap.
qemuio.write_pattern(vol.volumePath,
"qcow2",
offset=1 * MiB,
len=64 * KiB,
pattern=0xf1)
# Add bitmap 1 and write second cluster.
bitmap1 = str(uuid.uuid4())
qemuimg.bitmap_add(vol.volumePath, bitmap1).run()
qemuio.write_pattern(vol.volumePath,
"qcow2",
offset=2 * MiB,
len=64 * KiB,
pattern=0xf2)
# Add bitmap 2 and write third cluster.
bitmap2 = str(uuid.uuid4())
qemuimg.bitmap_add(vol.volumePath, bitmap2).run()
qemuio.write_pattern(vol.volumePath,
"qcow2",
offset=3 * MiB,
len=64 * KiB,
pattern=0xf3)
# Test bitmap 1 - recording changes since bitmap 1 was added.
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
"readonly": True,
"bitmap": bitmap1,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
dirty_extents = [(e.length, e.dirty)
for e in extents[c.dirty_bitmap]]
assert dirty_extents == [
(2 * MiB, False),
(64 * KiB, True),
(1 * MiB - 64 * KiB, False),
(64 * KiB, True),
(nbd_env.virtual_size - 3 * MiB - 64 * KiB, False),
]
assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
# Test bitmap 2 - recording changes since bitmap 2 was added.
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
"readonly": True,
"bitmap": bitmap2,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
dirty_extents = [(e.length, e.dirty)
for e in extents[c.dirty_bitmap]]
assert dirty_extents == [
(3 * MiB, False),
(64 * KiB, True),
(nbd_env.virtual_size - 3 * MiB - 64 * KiB, False),
]
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
def test_create_with_bitmaps(tmp_storage, tmp_repo, fake_access, fake_rescan,
tmp_db, fake_task, fake_sanlock, domain_version):
sd_uuid = str(uuid.uuid4())
dev = tmp_storage.create_device(20 * GiB)
lvm.createVG(sd_uuid, [dev], blockSD.STORAGE_UNREADY_DOMAIN_TAG, 128)
vg = lvm.getVG(sd_uuid)
dom = blockSD.BlockStorageDomain.create(sdUUID=sd_uuid,
domainName="domain",
domClass=sd.DATA_DOMAIN,
vgUUID=vg.uuid,
version=domain_version,
storageType=sd.ISCSI_DOMAIN)
sdCache.knownSDs[sd_uuid] = blockSD.findDomain
sdCache.manuallyAddDomain(dom)
dom.refresh()
dom.attach(tmp_repo.pool_id)
img_uuid = str(uuid.uuid4())
base_vol_uuid = str(uuid.uuid4())
base_vol_capacity = GiB
top_vol_uuid = str(uuid.uuid4())
vol_capacity = 2 * base_vol_capacity
bitmap_names = ['bitmap1', 'bitmap2']
# Create base volume.
dom.createVolume(imgUUID=img_uuid,
capacity=base_vol_capacity,
volFormat=sc.COW_FORMAT,
preallocate=sc.SPARSE_VOL,
diskType='DATA',
volUUID=base_vol_uuid,
desc="Test base volume",
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID)
base_vol = dom.produceVolume(img_uuid, base_vol_uuid)
base_vol_path = base_vol.getVolumePath()
# Prepare the volume in order to create bitmaps
base_vol.prepare()
# Add new bitmaps to base volume
for bitmap_name in bitmap_names:
op = qemuimg.bitmap_add(
base_vol_path,
bitmap_name,
)
op.run()
# Teardown the volume, test if prepare() will be
# called during the snapshot creation
base_vol.teardown(sd_uuid, base_vol_uuid)
# Create top volume with bitmaps.
dom.createVolume(imgUUID=img_uuid,
capacity=vol_capacity,
volFormat=sc.COW_FORMAT,
preallocate=sc.SPARSE_VOL,
diskType='DATA',
volUUID=top_vol_uuid,
desc="Test top volume",
srcImgUUID=base_vol.imgUUID,
srcVolUUID=base_vol.volUUID,
add_bitmaps=True)
top_vol = dom.produceVolume(img_uuid, top_vol_uuid)
top_vol_path = top_vol.getVolumePath()
# Prepare the volume in order to get
# info on the bitmaps
top_vol.prepare()
info = qemuimg.info(top_vol_path)
# Teardown top volume
base_vol.teardown(sd_uuid, top_vol_uuid)
assert info['format-specific']['data']['bitmaps'] == [
{
"flags": ["auto"],
"name": bitmap_names[0],
"granularity": 65536
},
{
"flags": ["auto"],
"name": bitmap_names[1],
"granularity": 65536
},
]
def test_create_volume_with_bitmaps(user_domain, local_fallocate):
if user_domain.getVersion() == 3:
pytest.skip("Bitmaps operations not supported in v3 domains")
parent_img_uuid = str(uuid.uuid4())
parent_vol_uuid = str(uuid.uuid4())
bitmap_names = ['bitmap1', 'bitmap2']
# Create base volume
user_domain.createVolume(
imgUUID=parent_img_uuid,
capacity=SPARSE_VOL_SIZE,
volFormat=sc.COW_FORMAT,
preallocate=sc.SPARSE_VOL,
diskType="DATA",
volUUID=parent_vol_uuid,
desc="test",
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID)
parent_vol = user_domain.produceVolume(
parent_img_uuid, parent_vol_uuid)
parent_vol_path = parent_vol.getVolumePath()
# Add new bitmaps to base volume
for bitmap_name in bitmap_names:
op = qemuimg.bitmap_add(
parent_vol_path,
bitmap_name,
)
op.run()
vol_uuid = str(uuid.uuid4())
# Create top volume
user_domain.createVolume(
imgUUID=parent_img_uuid,
capacity=SPARSE_VOL_SIZE,
volFormat=sc.COW_FORMAT,
preallocate=sc.SPARSE_VOL,
diskType='DATA',
volUUID=vol_uuid,
desc="Test volume",
srcImgUUID=parent_vol.imgUUID,
srcVolUUID=parent_vol.volUUID,
add_bitmaps=True
)
vol = user_domain.produceVolume(
parent_img_uuid, vol_uuid)
vol_path = vol.getVolumePath()
info = qemuimg.info(vol_path)
assert info['format-specific']['data']['bitmaps'] == [
{
"flags": ["auto"],
"name": bitmap_names[0],
"granularity": 65536
},
{
"flags": ["auto"],
"name": bitmap_names[1],
"granularity": 65536
},
]
