def test_qcow2_compat(self, env_type, qcow2_compat, sd_version):
src_fmt = sc.name2type("cow")
dst_fmt = sc.name2type("cow")
job_id = make_uuid()
with self.make_env(env_type,
src_fmt,
dst_fmt,
sd_version=sd_version,
src_qcow2_compat=qcow2_compat) as env:
src_vol = env.src_chain[0]
dst_vol = env.dst_chain[0]
source = dict(endpoint_type='div',
sd_id=src_vol.sdUUID,
img_id=src_vol.imgUUID,
vol_id=src_vol.volUUID)
dest = dict(endpoint_type='div',
sd_id=dst_vol.sdUUID,
img_id=dst_vol.imgUUID,
vol_id=dst_vol.volUUID)
job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest)
job.run()
wait_for_job(job)
actual_compat = qemuimg.info(dst_vol.volumePath)['compat']
self.assertEqual(actual_compat, env.sd_manifest.qcow2_compat())
def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt):
src_fmt = sc.name2type(src_fmt)
dst_fmt = sc.name2type(dst_fmt)
job_id = str(uuid.uuid4())
with self.get_vols(env_type, src_fmt, dst_fmt) as (src_chain,
dst_chain):
src_vol = src_chain[0]
dst_vol = dst_chain[0]
write_qemu_chain(src_chain)
self.assertRaises(ChainVerificationError,
verify_qemu_chain, dst_chain)
source = dict(endpoint_type='div', sd_id=src_vol.sdUUID,
img_id=src_vol.imgUUID, vol_id=src_vol.volUUID)
dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID,
img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID)
job = storage.sdm.api.copy_data.Job(job_id, None, source, dest)
job.run()
wait_for_job(job)
self.assertEqual(jobs.STATUS.DONE, job.status)
self.assertEqual(100.0, job.progress)
self.assertNotIn('error', job.info())
verify_qemu_chain(dst_chain)
self.assertEqual(sc.fmt2str(dst_fmt),
qemuimg.info(dst_vol.volumePath)['format'])
def test_optional_fields(self, qemu_field, info_field):
data = self._fake_info()
del data[qemu_field]
with MonkeyPatchScope([(commands, "execCmd",
partial(fake_json_call, data))]):
info = qemuimg.info('unused')
self.assertNotIn(info_field, info)
def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt):
src_fmt = sc.name2type(src_fmt)
dst_fmt = sc.name2type(dst_fmt)
job_id = make_uuid()
with self.make_env(env_type, src_fmt, dst_fmt) as env:
src_vol = env.src_chain[0]
dst_vol = env.dst_chain[0]
write_qemu_chain(env.src_chain)
self.assertRaises(ChainVerificationError,
verify_qemu_chain, env.dst_chain)
source = dict(endpoint_type='div', sd_id=src_vol.sdUUID,
img_id=src_vol.imgUUID, vol_id=src_vol.volUUID)
dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID,
img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID)
job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest)
job.run()
wait_for_job(job)
self.assertEqual(sorted(self.expected_locks(src_vol, dst_vol)),
sorted(guarded.context.locks))
self.assertEqual(jobs.STATUS.DONE, job.status)
self.assertEqual(100.0, job.progress)
self.assertNotIn('error', job.info())
verify_qemu_chain(env.dst_chain)
self.assertEqual(sc.fmt2str(dst_fmt),
qemuimg.info(dst_vol.volumePath)['format'])
def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt):
src_fmt = sc.name2type(src_fmt)
dst_fmt = sc.name2type(dst_fmt)
job_id = make_uuid()
with self.get_vols(env_type, src_fmt,
dst_fmt) as (src_chain, dst_chain):
src_vol = src_chain[0]
dst_vol = dst_chain[0]
write_qemu_chain(src_chain)
self.assertRaises(ChainVerificationError, verify_qemu_chain,
dst_chain)
source = dict(endpoint_type='div',
sd_id=src_vol.sdUUID,
img_id=src_vol.imgUUID,
vol_id=src_vol.volUUID)
dest = dict(endpoint_type='div',
sd_id=dst_vol.sdUUID,
img_id=dst_vol.imgUUID,
vol_id=dst_vol.volUUID)
job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest)
job.run()
wait_for_job(job)
self.assertEqual(sorted(self.expected_locks(src_vol, dst_vol)),
sorted(guarded.context.locks))
self.assertEqual(jobs.STATUS.DONE, job.status)
self.assertEqual(100.0, job.progress)
self.assertNotIn('error', job.info())
verify_qemu_chain(dst_chain)
self.assertEqual(sc.fmt2str(dst_fmt),
qemuimg.info(dst_vol.volumePath)['format'])
def test_finalize(self, sd_type, chain_len, base_index, top_index):
with self.make_env(sd_type=sd_type, chain_len=chain_len) as env:
base_vol = env.chain[base_index]
# This volume *was* prepared
base_vol.setLegality(sc.ILLEGAL_VOL)
top_vol = env.chain[top_index]
subchain_info = dict(
sd_id=base_vol.sdUUID,
img_id=base_vol.imgUUID,
base_id=base_vol.volUUID,
top_id=top_vol.volUUID,
base_generation=0,
)
subchain = merge.SubchainInfo(subchain_info, 0)
merge.finalize(subchain)
# If top has a child, the child must now be rebased on base.
if top_vol is not env.chain[-1]:
child_vol = env.chain[top_index + 1]
info = qemuimg.info(child_vol.volumePath)
self.assertEqual(info["backingfile"], base_vol.volumePath)
# verify syncVolumeChain arguments
self.assertEquals(image.Image.syncVolumeChain.sd_id, subchain.sd_id)
self.assertEquals(image.Image.syncVolumeChain.img_id, subchain.img_id)
self.assertEquals(image.Image.syncVolumeChain.vol_id, env.chain[-1].volUUID)
new_chain = [vol.volUUID for vol in env.chain]
new_chain.remove(top_vol.volUUID)
self.assertEquals(image.Image.syncVolumeChain.actual_chain, new_chain)
self.assertEqual(base_vol.getLegality(), sc.LEGAL_VOL)
def test_optional_fields(self, qemu_field, info_field):
data = self._fake_info()
del data[qemu_field]
with MonkeyPatchScope([(commands, "execCmd",
partial(fake_json_call, data))]):
info = qemuimg.info('unused')
self.assertNotIn(info_field, info)
def invalid_vm_conf_disk(vol):
"""
set VM metadata images format to RAW
Since commit 0b61c4851a528fd6354d9ab77a68085c41f35dc9 copy of internal raw
volumes is done using 'qemu-img convert' instead of invoking 'dd'.
Consequently, exporting VM metadata images (produced during live snapshot)
fails on qemu-img convert - since the images 'impersonate' to qcow2 (the
format in .meta file is cow, whereas the real format is raw). This problem
is documented by https://bugzilla.redhat.com/1282239 and has subsequently
been fixed in ovirt-engine (see https://gerrit.ovirt.org/48768).
Since VM metadata volumes with this problem may still exist in storage we
must keep using this workaround to avoid problems with copying VM disks.
"""
if vol.getFormat() == sc.COW_FORMAT and vol.getSize() == VM_CONF_SIZE_BLK:
info = qemuimg.info(vol.getVolumePath())
actual_format = info['format']
if actual_format == qemuimg.FORMAT.RAW:
log.warning(
"Incorrect volume format %r has been detected"
" for volume %r, using the actual format %r.",
qemuimg.FORMAT.QCOW2, vol.volUUID, qemuimg.FORMAT.RAW)
return True
return False
def invalid_vm_conf_disk(vol):
"""
set VM metadata images format to RAW
Since commit 0b61c4851a528fd6354d9ab77a68085c41f35dc9 copy of internal raw
volumes is done using 'qemu-img convert' instead of invoking 'dd'.
Consequently, exporting VM metadata images (produced during live snapshot)
fails on qemu-img convert - since the images 'impersonate' to qcow2 (the
format in .meta file is cow, whereas the real format is raw). This problem
is documented by https://bugzilla.redhat.com/1282239 and has subsequently
been fixed in ovirt-engine (see https://gerrit.ovirt.org/48768).
Since VM metadata volumes with this problem may still exist in storage we
must keep using this workaround to avoid problems with copying VM disks.
"""
if vol.getFormat() == sc.COW_FORMAT and vol.getSize() == VM_CONF_SIZE_BLK:
info = qemuimg.info(vol.getVolumePath())
actual_format = info['format']
if actual_format == qemuimg.FORMAT.RAW:
log.warning("Incorrect volume format %r has been detected"
" for volume %r, using the actual format %r.",
qemuimg.FORMAT.QCOW2,
vol.volUUID,
qemuimg.FORMAT.RAW)
return True
return False
def test_finalize(self, sd_type, chain_len, base_index, top_index):
with self.make_env(sd_type=sd_type, chain_len=chain_len) as env:
base_vol = env.chain[base_index]
top_vol = env.chain[top_index]
subchain_info = dict(sd_id=base_vol.sdUUID,
img_id=base_vol.imgUUID,
base_id=base_vol.volUUID,
top_id=top_vol.volUUID,
base_generation=0)
subchain = merge.SubchainInfo(subchain_info, 0)
merge.finalize(subchain)
# If top has a child, the child must now be rebased on base.
if top_vol is not env.chain[-1]:
child_vol = env.chain[top_index + 1]
info = qemuimg.info(child_vol.volumePath)
self.assertEqual(
info['backingfile'],
volume.getBackingVolumePath(subchain.img_id,
subchain.base_id))
# verify syncVolumeChain arguments
self.check_sync_volume_chain(subchain, env.chain[-1].volUUID)
new_chain = [vol.volUUID for vol in env.chain]
new_chain.remove(top_vol.volUUID)
self.assertEqual(image.Image.syncVolumeChain.actual_chain,
new_chain)
self.assertEqual(base_vol.getLegality(), sc.LEGAL_VOL)
def test_qemuimg_info(self, vol_format, qemu_format):
with self.fake_env() as env:
artifacts = env.sd_manifest.get_volume_artifacts(
self.img_id, self.vol_id)
size, vol_format, disk_type, desc = BASE_PARAMS[vol_format]
artifacts.create(size, vol_format, disk_type, desc)
artifacts.commit()
info = qemuimg.info(artifacts.volume_path)
self.assertEqual(qemu_format, info['format'])
self.assertEqual(size, info['virtualsize'])
self.assertNotIn('backingfile', info)
def test_empty_image(self, qcow2_compat, desired_qcow2_compat):
with namedTemporaryDir() as tmpdir:
base_path = os.path.join(tmpdir, 'base.img')
leaf_path = os.path.join(tmpdir, 'leaf.img')
size = 1048576
qemuimg.create(base_path, size=size, format=qemuimg.FORMAT.RAW)
qemuimg.create(leaf_path, format=qemuimg.FORMAT.QCOW2,
backing=base_path)
qemuimg.amend(leaf_path, desired_qcow2_compat)
self.assertEquals(qemuimg.info(leaf_path)['compat'],
desired_qcow2_compat)
def test_empty_image(self, qcow2_compat, desired_qcow2_compat):
with namedTemporaryDir() as tmpdir:
base_path = os.path.join(tmpdir, 'base.img')
leaf_path = os.path.join(tmpdir, 'leaf.img')
size = 1048576
qemuimg.create(base_path, size=size, format=qemuimg.FORMAT.RAW)
qemuimg.create(leaf_path, format=qemuimg.FORMAT.QCOW2,
backing=base_path)
qemuimg.amend(leaf_path, desired_qcow2_compat)
self.assertEquals(qemuimg.info(leaf_path)['compat'],
desired_qcow2_compat)
def test_qemuimg_info(self, vol_format, qemu_format):
with self.fake_env() as env:
artifacts = env.sd_manifest.get_volume_artifacts(
self.img_id, self.vol_id)
size, vol_format, disk_type, desc = BASE_PARAMS[vol_format]
artifacts.create(size, vol_format, disk_type, desc)
artifacts.commit()
info = qemuimg.info(artifacts.volume_path)
self.assertEqual(qemu_format, info['format'])
self.assertEqual(size, info['virtualsize'])
self.assertNotIn('backingfile', info)
def test_compat_reported_for_qcow2_only(self):
data = {
"virtual-size": 1048576,
"filename": "raw.img",
"format": "raw",
"actual-size": 0,
"dirty-flag": False
}
with MonkeyPatchScope([(commands, "execCmd",
partial(fake_json_call, data))]):
info = qemuimg.info('unused')
self.assertNotIn('compat', info)
def test_qemu1_no_backing_file(self):
def call(cmd, **kw):
out = [
"image: leaf.img", "file format: qcow2",
"virtual size: 1.0G (1073741824 bytes)", "disk size: 196K",
"cluster_size: 65536"
]
return 0, out, []
with MonkeyPatchScope([(utils, "execCmd", call)]):
info = qemuimg.info('leaf.img')
self.assertNotIn('backingfile', info)
def updateInvalidatedSize(self):
# During some complex flows the volume size might have been marked as
# invalidated (e.g. during a transaction). Here we are checking
# NOTE: the prerequisite to run this is that the volume is accessible
# (e.g. lv active) and not in use by another process (e.g. dd, qemu).
# Going directly to the metadata parameter as we should skip the size
# validation in getSize.
if int(self.getMetaParam(SIZE)) < 1:
volInfo = qemuimg.info(
self.getVolumePath(), fmt2str(self.getFormat()))
# qemu/qemu-img rounds down
self.setSize(volInfo['virtualsize'] / BLOCK_SIZE)
def test_compat_reported_for_qcow2_only(self):
data = {
"virtual-size": 1048576,
"filename": "raw.img",
"format": "raw",
"actual-size": 0,
"dirty-flag": False
}
with MonkeyPatchScope([(commands, "execCmd",
partial(fake_json_call, data))]):
info = qemuimg.info('unused')
self.assertNotIn('compat', info)
def test_qemu1_no_backing_file(self):
def call(cmd, **kw):
out = ["image: leaf.img",
"file format: qcow2",
"virtual size: 1.0G (1073741824 bytes)",
"disk size: 196K",
"cluster_size: 65536"]
return 0, out, []
with MonkeyPatchScope([(utils, "execCmd", call)]):
info = qemuimg.info('leaf.img')
self.assertNotIn('backingfile', info)
def updateInvalidatedSize(self):
# During some complex flows the volume size might have been marked as
# invalidated (e.g. during a transaction). Here we are checking
# NOTE: the prerequisite to run this is that the volume is accessible
# (e.g. lv active) and not in use by another process (e.g. dd, qemu).
# Going directly to the metadata parameter as we should skip the size
# validation in getSize.
if int(self.getMetaParam(sc.SIZE)) < 1:
volInfo = qemuimg.info(self.getVolumePath(),
sc.fmt2str(self.getFormat()))
# qemu/qemu-img rounds down
self.setSize(volInfo['virtualsize'] / sc.BLOCK_SIZE)
def test_qemu1_backing(self):
def call(cmd, **kw):
out = ["image: leaf.img",
"file format: qcow2",
"virtual size: 1.0G (1073741824 bytes)",
"disk size: 196K",
"cluster_size: 65536",
"backing file: base.img (actual path: /tmp/base.img)"]
return 0, out, []
with MonkeyPatchScope([(utils, "execCmd", call)]):
info = qemuimg.info('leaf.img')
self.assertEquals('base.img', info['backingfile'])
def test_qemu1_backing(self):
def call(cmd, **kw):
out = [
"image: leaf.img", "file format: qcow2",
"virtual size: 1.0G (1073741824 bytes)", "disk size: 196K",
"cluster_size: 65536",
"backing file: base.img (actual path: /tmp/base.img)"
]
return 0, out, []
with MonkeyPatchScope([(utils, "execCmd", call)]):
info = qemuimg.info('leaf.img')
self.assertEquals('base.img', info['backingfile'])
def test_qemu2_backing_no_cluster(self):
def call(cmd, **kw):
out = [
"image: leaf.img", "file format: qcow2",
"virtual size: 1.0G (1073741824 bytes)", "disk size: 196K",
"backing file: base.img (actual path: /tmp/base.img)",
"Format specific information:", " compat: 1.1",
" lazy refcounts: false"
]
return 0, out, []
with MonkeyPatchScope([(utils, "execCmd", call)]):
info = qemuimg.info('leaf.img')
self.assertEquals('base.img', info['backingfile'])
def test_qemu2_backing_no_cluster(self):
def call(cmd, **kw):
out = ["image: leaf.img",
"file format: qcow2",
"virtual size: 1.0G (1073741824 bytes)",
"disk size: 196K",
"backing file: base.img (actual path: /tmp/base.img)",
"Format specific information:",
" compat: 1.1",
" lazy refcounts: false"]
return 0, out, []
with MonkeyPatchScope([(utils, "execCmd", call)]):
info = qemuimg.info('leaf.img')
self.assertEquals('base.img', info['backingfile'])
def test_info(self):
with namedTemporaryDir() as tmpdir:
base_path = os.path.join(tmpdir, 'base.img')
leaf_path = os.path.join(tmpdir, 'leaf.img')
size = 1048576
leaf_fmt = qemuimg.FORMAT.QCOW2
with MonkeyPatchScope([(qemuimg, 'config', CONFIG)]):
qemuimg.create(base_path, size=size, format=qemuimg.FORMAT.RAW)
qemuimg.create(leaf_path, format=leaf_fmt, backing=base_path)
info = qemuimg.info(leaf_path)
self.assertEqual(leaf_fmt, info['format'])
self.assertEqual(size, info['virtualsize'])
self.assertEqual(self.CLUSTER_SIZE, info['clustersize'])
self.assertEqual(base_path, info['backingfile'])
self.assertEqual('0.10', info['compat'])
def test_qemu2_no_backing_file(self):
def call(cmd, **kw):
out = [
"image: leaf.img", "file format: qcow2",
"virtual size: 1.0G (1073741824 bytes)", "disk size: 196K",
"cluster_size: 65536", "Format specific information:",
" compat: 1.1", " lazy refcounts: false"
]
return 0, out, []
with MonkeyPatchScope([(utils, "execCmd", call)]):
info = qemuimg.info('leaf.img')
self.assertEquals('qcow2', info['format'])
self.assertEquals(1073741824, info['virtualsize'])
self.assertEquals(65536, info['clustersize'])
self.assertNotIn('backingfile', info)
def test_info(self):
with namedTemporaryDir() as tmpdir:
base_path = os.path.join(tmpdir, 'base.img')
leaf_path = os.path.join(tmpdir, 'leaf.img')
size = 1048576
leaf_fmt = qemuimg.FORMAT.QCOW2
with MonkeyPatchScope([(qemuimg, 'config', CONFIG)]):
qemuimg.create(base_path, size=size, format=qemuimg.FORMAT.RAW)
qemuimg.create(leaf_path, format=leaf_fmt, backing=base_path)
info = qemuimg.info(leaf_path)
self.assertEqual(leaf_fmt, info['format'])
self.assertEqual(size, info['virtualsize'])
self.assertEqual(self.CLUSTER_SIZE, info['clustersize'])
self.assertEqual(base_path, info['backingfile'])
self.assertEqual('0.10', info['compat'])
def test_qemu2_no_backing_file(self):
def call(cmd, **kw):
out = ["image: leaf.img",
"file format: qcow2",
"virtual size: 1.0G (1073741824 bytes)",
"disk size: 196K",
"cluster_size: 65536",
"Format specific information:",
" compat: 1.1",
" lazy refcounts: false"]
return 0, out, []
with MonkeyPatchScope([(utils, "execCmd", call)]):
info = qemuimg.info('leaf.img')
self.assertEquals('qcow2', info['format'])
self.assertEquals(1073741824, info['virtualsize'])
self.assertEquals(65536, info['clustersize'])
self.assertNotIn('backingfile', info)
def v3ResetMetaVolSize(vol):
# BZ811880 Verifiying that the volume size is the same size advertised
# by the metadata
log.debug("Checking the volume size for the volume %s", vol.volUUID)
metaVolSize = int(vol.getMetaParam(volume.SIZE))
if vol.getFormat() == volume.COW_FORMAT:
qemuVolInfo = qemuimg.info(vol.getVolumePath(),
qemuimg.FORMAT.QCOW2)
virtVolSize = qemuVolInfo["virtualsize"] / V2META_SECTORSIZE
else:
virtVolSize = vol.getVolumeSize()
if metaVolSize != virtVolSize:
log.warn("Fixing the mismatch between the metadata volume size "
"(%s) and the volume virtual size (%s) for the volume "
"%s", vol.volUUID, metaVolSize, virtVolSize)
vol.setMetaParam(volume.SIZE, str(virtVolSize))
def estimate_size(filename):
"""
Estimating qcow2 file size once converted from raw to qcow2.
The filename is a path (sparse or preallocated),
or a path to preallocated block device.
"""
info = qemuimg.info(filename)
if (info['format'] != qemuimg.FORMAT.RAW):
raise ValueError("Estimate size is only supported for raw format. file"
" %s is with format %s" % (filename, info['format']))
# Get used clusters and virtual size of destination volume.
virtual_size = info['virtualsize']
meta_size = _estimate_metadata_size(virtual_size)
runs = qemuimg.map(filename)
used_clusters = count_clusters(runs)
# Return the estimated size.
return meta_size + used_clusters * CLUSTER_SIZE
def test_qcow2_compat(self, env_type, qcow2_compat, sd_version):
src_fmt = sc.name2type("cow")
dst_fmt = sc.name2type("cow")
job_id = make_uuid()
with self.make_env(env_type, src_fmt, dst_fmt, sd_version=sd_version,
src_qcow2_compat=qcow2_compat) as env:
src_vol = env.src_chain[0]
dst_vol = env.dst_chain[0]
source = dict(endpoint_type='div', sd_id=src_vol.sdUUID,
img_id=src_vol.imgUUID, vol_id=src_vol.volUUID)
dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID,
img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID)
job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest)
job.run()
wait_for_job(job)
actual_compat = qemuimg.info(dst_vol.volumePath)['compat']
self.assertEqual(actual_compat, env.sd_manifest.qcow2_compat())
def getQemuImageInfo(self):
"""
Returns volume information as returned by qemu-img info command
"""
return qemuimg.info(self.getVolumePath(), sc.fmt2str(self.getFormat()))
def getQemuImageInfo(self):
"""
Returns volume information as returned by qemu-img info command
"""
return qemuimg.info(self.getVolumePath(),
sc.fmt2str(self.getFormat()))