def test_convert_from_v3_to_v4_localfs(tmpdir, tmp_repo, fake_access):
remote_path = str(tmpdir.mkdir("domain"))
tmp_repo.connect_localfs(remote_path)
dom = localFsSD.LocalFsStorageDomain.create(sdUUID=str(uuid.uuid4()),
domainName="domain",
domClass=sd.DATA_DOMAIN,
remotePath=remote_path,
version=3,
storageType=sd.LOCALFS_DOMAIN,
block_size=sc.BLOCK_SIZE_512,
alignment=sc.ALIGNMENT_1M)
assert dom.getVersion() == 3
fc = formatconverter.DefaultFormatConverter()
fc.convert(repoPath=tmp_repo.path,
hostId=1,
imageRepo=dom,
isMsd=False,
targetFormat='4')
# LocalFS do not support external leases, so the only change is the
# version.
assert dom.getVersion() == 4
def test_convert_from_v3_to_v4_localfs(tmp_repo, fake_access):
# In a real domain this is a symlink to the domain directory.
remote_path = "/data/domain"
dom_dir = os.path.join(
sc.REPO_MOUNT_DIR, fileUtils.transformPath(remote_path))
os.makedirs(dom_dir)
dom = localFsSD.LocalFsStorageDomain.create(
sdUUID=str(uuid.uuid4()),
domainName="domain",
domClass=sd.DATA_DOMAIN,
remotePath=remote_path,
version=3,
storageType=sd.LOCALFS_DOMAIN,
block_size=sc.BLOCK_SIZE_512,
alignment=sc.ALIGN_1M)
assert dom.getVersion() == 3
fc = formatconverter.DefaultFormatConverter()
fc.convert(
repoPath=tmp_repo.path,
hostId=1,
imageRepo=dom,
isMsd=False,
targetFormat='4')
# LocalFS do not support external leases, so the only change is the
# version.
assert dom.getVersion() == 4
def test_convert_from_v4_to_v5_localfs(tmpdir, tmp_repo, tmp_db, fake_access,
fake_rescan, fake_task):
dom = tmp_repo.create_localfs_domain(name="domain", version=4)
# Create some volumes in v4 format.
for i in range(3):
dom.createVolume(desc="Awesome volume %d" % i,
diskType="DATA",
imgUUID=str(uuid.uuid4()),
preallocate=sc.SPARSE_VOL,
size=10 * 1024**3,
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID,
volFormat=sc.COW_FORMAT,
volUUID=str(uuid.uuid4()))
# Record domain and volumes metadata before conversion.
old_dom_md = dom.getMetadata()
volumes_md = {vol.volUUID: vol.getMetadata() for vol in dom.iter_volumes()}
fc = formatconverter.DefaultFormatConverter()
fc.convert(repoPath=tmp_repo.path,
hostId=1,
imageRepo=dom,
isMsd=False,
targetFormat='5')
# Verify changes in domain metadata.
new_dom_md = dom.getMetadata()
# Values modified in v5.
assert old_dom_md.pop("VERSION") == 4
assert new_dom_md.pop("VERSION") == 5
# Keys added in v5.
assert new_dom_md.pop("BLOCK_SIZE") == sc.BLOCK_SIZE_512
assert new_dom_md.pop("ALIGNMENT") == sc.ALIGNMENT_1M
# Rest of the values should not be modified.
assert new_dom_md == old_dom_md
# Verify that volumes metadata was converted to v5 format.
for vol in dom.iter_volumes():
vol_md = volumes_md[vol.volUUID]
meta_path = vol.getMetaVolumePath()
with open(meta_path) as f:
data = f.read()
assert data == vol_md.storage_format(5)
def test_convert_from_v3_to_v4_localfs(tmpdir, tmp_repo, fake_access):
dom = tmp_repo.create_localfs_domain(name="domain", version=3)
assert dom.getVersion() == 3
fc = formatconverter.DefaultFormatConverter()
fc.convert(repoPath=tmp_repo.path,
hostId=1,
imageRepo=dom,
isMsd=False,
targetFormat='4')
# LocalFS do not support external leases, so the only change is the
# version.
assert dom.getVersion() == 4
def test_convert_to_v5_localfs(tmpdir, tmp_repo, tmp_db, fake_access,
fake_rescan, fake_task, src_version):
dom = tmp_repo.create_localfs_domain(name="domain", version=src_version)
# Create some volumes in v4 format.
for i in range(3):
dom.createVolume(desc="Awesome volume %d" % i,
diskType="DATA",
imgUUID=str(uuid.uuid4()),
preallocate=sc.SPARSE_VOL,
capacity=10 * GiB,
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID,
volFormat=sc.COW_FORMAT,
volUUID=str(uuid.uuid4()))
# Record domain and volumes metadata before conversion.
old_dom_md = dom.getMetadata()
volumes_md = {vol.volUUID: vol.getMetadata() for vol in dom.iter_volumes()}
# Simulate a partly-deleted volume with cleared metada. Such volumes could
# be created by vdsm < 4.20.34-1.
img_id = str(uuid.uuid4())
vol_id = str(uuid.uuid4())
dom.createVolume(desc="Half deleted volume",
diskType="DATA",
imgUUID=img_id,
preallocate=sc.SPARSE_VOL,
capacity=10 * GiB,
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID,
volFormat=sc.COW_FORMAT,
volUUID=vol_id)
partly_deleted_vol = dom.produceVolume(img_id, vol_id)
meta_path = partly_deleted_vol.getMetaVolumePath()
with open(meta_path, "wb") as f:
f.write(CLEARED_VOLUME_METADATA)
# Simulate a volume with invalid metada to make sure such volume will not
# break conversion.
img_id = str(uuid.uuid4())
vol_id = str(uuid.uuid4())
dom.createVolume(desc="Volume with invalid metadata",
diskType="DATA",
imgUUID=img_id,
preallocate=sc.SPARSE_VOL,
capacity=10 * GiB,
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID,
volFormat=sc.COW_FORMAT,
volUUID=vol_id)
invalid_md_vol = dom.produceVolume(img_id, vol_id)
meta_path = invalid_md_vol.getMetaVolumePath()
with open(meta_path, "wb") as f:
f.write(INVALID_VOLUME_METADATA)
# These volumes will not be converted to V5 format.
skip_volumes = {partly_deleted_vol.volUUID, invalid_md_vol.volUUID}
# Convert the domain.
fc = formatconverter.DefaultFormatConverter()
fc.convert(repoPath=tmp_repo.path,
hostId=1,
imageRepo=dom,
isMsd=False,
targetFormat='5')
# Verify changes in domain metadata.
new_dom_md = dom.getMetadata()
# Values modified in v5.
assert old_dom_md.pop("VERSION") == src_version
assert new_dom_md.pop("VERSION") == 5
# Keys added in v5.
assert new_dom_md.pop("BLOCK_SIZE") == sc.BLOCK_SIZE_512
assert new_dom_md.pop("ALIGNMENT") == sc.ALIGNMENT_1M
# Rest of the values should not be modified.
assert new_dom_md == old_dom_md
# Verify that volumes metadata was converted to v5 format.
for vol in dom.iter_volumes():
if vol.volUUID in skip_volumes:
continue
vol_md = volumes_md[vol.volUUID]
meta_path = vol.getMetaVolumePath()
with open(meta_path, "rb") as f:
data = f.read()
assert data == vol_md.storage_format(5)
# Verify that invalid metadata was left without change.
meta_path = partly_deleted_vol.getMetaVolumePath()
with open(meta_path, "rb") as f:
assert f.read() == CLEARED_VOLUME_METADATA
meta_path = invalid_md_vol.getMetaVolumePath()
with open(meta_path, "rb") as f:
assert f.read() == INVALID_VOLUME_METADATA
def test_convert_to_v5_block(tmpdir, tmp_repo, tmp_storage, tmp_db,
fake_rescan, fake_task, fake_sanlock,
src_version):
sd_uuid = str(uuid.uuid4())
dev = tmp_storage.create_device(20 * GiB)
lvm.createVG(sd_uuid, [dev], blockSD.STORAGE_UNREADY_DOMAIN_TAG, 128)
vg = lvm.getVG(sd_uuid)
dom = blockSD.BlockStorageDomain.create(sdUUID=sd_uuid,
domainName="domain",
domClass=sd.DATA_DOMAIN,
vgUUID=vg.uuid,
version=src_version,
storageType=sd.ISCSI_DOMAIN)
sdCache.knownSDs[sd_uuid] = blockSD.findDomain
sdCache.manuallyAddDomain(dom)
# Create domain directory structure.
dom.refresh()
# Only attached domains are converted.
dom.attach(tmp_repo.pool_id)
# Create some volumes in v4 format.
for i in range(3):
dom.createVolume(desc="Awesome volume %d" % i,
diskType="DATA",
imgUUID=str(uuid.uuid4()),
preallocate=sc.SPARSE_VOL,
capacity=10 * GiB,
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID,
volFormat=sc.COW_FORMAT,
volUUID=str(uuid.uuid4()))
# Record domain and volumes metadata before conversion.
old_dom_md = dom.getMetadata()
volumes_md = {vol.volUUID: vol.getMetadata() for vol in dom.iter_volumes()}
# Simulate a partly-deleted volume with cleared metada. Such volumes could
# be created by vdsm < 4.20.34-1.
img_id = str(uuid.uuid4())
vol_id = str(uuid.uuid4())
dom.createVolume(desc="Half deleted volume",
diskType="DATA",
imgUUID=img_id,
preallocate=sc.SPARSE_VOL,
capacity=10 * GiB,
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID,
volFormat=sc.COW_FORMAT,
volUUID=vol_id)
partly_deleted_vol = dom.produceVolume(img_id, vol_id)
slot = partly_deleted_vol.getMetadataId()[1]
dom.manifest.write_metadata_block(slot, CLEARED_VOLUME_METADATA)
# Simulate a volume with invalid metada to make sure such volume will not
# break conversion.
img_id = str(uuid.uuid4())
vol_id = str(uuid.uuid4())
dom.createVolume(desc="Volume with invalid metadata",
diskType="DATA",
imgUUID=img_id,
preallocate=sc.SPARSE_VOL,
capacity=10 * GiB,
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID,
volFormat=sc.COW_FORMAT,
volUUID=vol_id)
invalid_md_vol = dom.produceVolume(img_id, vol_id)
slot = invalid_md_vol.getMetadataId()[1]
dom.manifest.write_metadata_block(slot, INVALID_VOLUME_METADATA)
# These volumes will not be converted to V5 format.
skip_volumes = {partly_deleted_vol.volUUID, invalid_md_vol.volUUID}
# Convert the domain.
fc = formatconverter.DefaultFormatConverter()
fc.convert(repoPath=tmp_repo.path,
hostId=1,
imageRepo=dom,
isMsd=False,
targetFormat='5')
# Verify changes in domain metadata.
new_dom_md = dom.getMetadata()
# Keys modified in v5.
assert old_dom_md.pop("VERSION") == src_version
assert new_dom_md.pop("VERSION") == 5
# Keys added in V5.
assert new_dom_md.pop("BLOCK_SIZE") == sc.BLOCK_SIZE_512
assert new_dom_md.pop("ALIGNMENT") == sc.ALIGNMENT_1M
# Kyes removed in v5.
assert old_dom_md.pop("LOGBLKSIZE") == sc.BLOCK_SIZE_512
assert old_dom_md.pop("PHYBLKSIZE") == sc.BLOCK_SIZE_512
# Rest of the keys must not be modifed by conversion.
assert old_dom_md == new_dom_md
# Verify that xleases volume is created when upgrading from version < 4.
xleases_vol = lvm.getLV(sd_uuid, sd.XLEASES)
assert int(xleases_vol.size) == sd.XLEASES_SLOTS * dom.alignment
with pytest.raises(se.NoSuchLease):
dom.manifest.lease_info("no-such-lease")
# Verify that volumes metadta was converted to v5 format.
for vol in dom.iter_volumes():
if vol.volUUID in skip_volumes:
continue
vol_md = volumes_md[vol.volUUID]
_, slot = vol.getMetadataId()
data = dom.manifest.read_metadata_block(slot)
data = data.rstrip(b"\0")
assert data == vol_md.storage_format(5)
# Verify that invalid metadata was copied to v5 area.
slot = partly_deleted_vol.getMetadataId()[1]
assert dom.manifest.read_metadata_block(slot) == CLEARED_VOLUME_METADATA
slot = invalid_md_vol.getMetadataId()[1]
assert dom.manifest.read_metadata_block(slot) == INVALID_VOLUME_METADATA
# Check that v4 metadata area is zeroed.
meta_path = dom.manifest.metadata_volume_path()
offset = blockSD.METADATA_BASE_V4
size = blockSD.METADATA_BASE_V5 - blockSD.METADATA_BASE_V4
data = misc.readblock(meta_path, offset, size)
assert data == b"\0" * size
def test_convert_from_v4_to_v5_block(tmpdir, tmp_repo, tmp_storage, tmp_db,
fake_rescan, fake_task, fake_sanlock):
sd_uuid = str(uuid.uuid4())
dev = tmp_storage.create_device(20 * 1024**3)
lvm.createVG(sd_uuid, [dev], blockSD.STORAGE_UNREADY_DOMAIN_TAG, 128)
vg = lvm.getVG(sd_uuid)
dom = blockSD.BlockStorageDomain.create(sdUUID=sd_uuid,
domainName="domain",
domClass=sd.DATA_DOMAIN,
vgUUID=vg.uuid,
version=4,
storageType=sd.ISCSI_DOMAIN,
block_size=sc.BLOCK_SIZE_512,
alignment=sc.ALIGNMENT_1M)
sdCache.knownSDs[sd_uuid] = blockSD.findDomain
sdCache.manuallyAddDomain(dom)
# Create domain directory structure.
dom.refresh()
# Only attached domains are converted.
dom.attach(tmp_repo.pool_id)
# Create some volumes in v4 format.
for i in range(3):
dom.createVolume(desc="Awesome volume %d" % i,
diskType="DATA",
imgUUID=str(uuid.uuid4()),
preallocate=sc.SPARSE_VOL,
size=10 * 1024**3,
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID,
volFormat=sc.COW_FORMAT,
volUUID=str(uuid.uuid4()))
# Record domain and volumes metadata before conversion.
old_dom_md = dom.getMetadata()
volumes_md = {vol.volUUID: vol.getMetadata() for vol in dom.iter_volumes()}
fc = formatconverter.DefaultFormatConverter()
fc.convert(repoPath=tmp_repo.path,
hostId=1,
imageRepo=dom,
isMsd=False,
targetFormat='5')
# Verify changes in domain metadata.
new_dom_md = dom.getMetadata()
# Keys modified in v5.
assert old_dom_md.pop("VERSION") == 4
assert new_dom_md.pop("VERSION") == 5
# Keys added in V5.
assert new_dom_md.pop("BLOCK_SIZE") == sc.BLOCK_SIZE_512
assert new_dom_md.pop("ALIGNMENT") == sc.ALIGNMENT_1M
# Kyes removed in v5.
assert old_dom_md.pop("LOGBLKSIZE") == sc.BLOCK_SIZE_512
assert old_dom_md.pop("PHYBLKSIZE") == sc.BLOCK_SIZE_512
# Rest of the keys must not be modifed by conversion.
assert old_dom_md == new_dom_md
# Verify that volumes metadta was converted to v5 format.
for vol in dom.iter_volumes():
vol_md = volumes_md[vol.volUUID]
_, slot = vol.getMetadataId()
data = dom.manifest.read_metadata_block(slot)
data = data.rstrip("\0")
assert data == vol_md.storage_format(5)
# Check that v4 metadata area is zeroed.
meta_path = dom.manifest.metadata_volume_path()
offset = blockSD.METADATA_BASE_V4
size = blockSD.METADATA_BASE_V5 - blockSD.METADATA_BASE_V4
data = misc.readblock(meta_path, offset, size)
assert data == "\0" * size
