def rebaseVolumeRollback(cls, taskObj, sdUUID, srcImg, srcVol, dstFormat,
srcParent, unsafe):
"""
Rebase volume rollback
"""
cls.log.info(
'rebase volume rollback (sdUUID=%s srcImg=%s srcVol=%s '
'dstFormat=%s srcParent=%s)', sdUUID, srcImg, srcVol, dstFormat,
srcParent)
imageResourcesNamespace = sd.getNamespace(sc.IMAGE_NAMESPACE, sdUUID)
with rm.acquireResource(imageResourcesNamespace, srcImg, rm.EXCLUSIVE):
vol = sdCache.produce(sdUUID).produceVolume(srcImg, srcVol)
vol.prepare(rw=True, chainrw=True, setrw=True)
volumePath = vol.getVolumePath()
backingVolPath = getBackingVolumePath(srcImg, srcParent)
try:
qemuimg.rebase(volumePath, backingVolPath,
sc.fmt2str(vol.getFormat()),
sc.fmt2str(int(dstFormat)),
misc.parseBool(unsafe), vars.task.aborting)
vol.setParent(srcParent)
vol.recheckIfLeaf()
except qemuimg.QImgError:
cls.log.exception(
'cannot rollback rebase for volume %s on '
'%s', volumePath, backingVolPath)
raise se.MergeVolumeRollbackError(srcVol)
finally:
vol.teardown(sdUUID, srcVol)
def test_chain_after_finalize(self, base_fmt):
with self.make_env(format=base_fmt, chain_len=3) as env:
base_vol = env.chain[0]
# We write data to the base and will read it from the child volume
# to verify that the chain is valid after qemu-rebase.
offset = 0
pattern = 0xf0
length = 1024
qemuio.write_pattern(base_vol.volumePath,
sc.fmt2str(base_vol.getFormat()),
offset=offset,
len=length,
pattern=pattern)
top_vol = env.chain[1]
child_vol = env.chain[2]
subchain_info = dict(sd_id=base_vol.sdUUID,
img_id=base_vol.imgUUID,
base_id=base_vol.volUUID,
top_id=top_vol.volUUID,
base_generation=0)
subchain = merge.SubchainInfo(subchain_info, 0)
merge.finalize(subchain)
qemuio.verify_pattern(child_vol.volumePath,
sc.fmt2str(child_vol.getFormat()),
offset=offset,
len=length,
pattern=pattern)
def clone(self, dstPath, volFormat):
"""
Clone self volume to the specified dst_image_dir/dst_volUUID
"""
wasleaf = False
taskName = "parent volume rollback: " + self.volUUID
vars.task.pushRecovery(
task.Recovery(taskName, "volume", "Volume",
"parentVolumeRollback",
[self.sdUUID, self.imgUUID, self.volUUID]))
if self.isLeaf():
wasleaf = True
self.setInternal()
try:
self.prepare(rw=False)
self.log.debug('cloning volume %s to %s', self.volumePath,
dstPath)
parent = getBackingVolumePath(self.imgUUID, self.volUUID)
domain = sdCache.produce(self.sdUUID)
qemuimg.create(dstPath, backing=parent,
format=sc.fmt2str(volFormat),
qcow2Compat=domain.qcow2_compat(),
backingFormat=sc.fmt2str(self.getFormat()))
self.teardown(self.sdUUID, self.volUUID)
except Exception as e:
self.log.exception('cannot clone image %s volume %s to %s',
self.imgUUID, self.volUUID, dstPath)
# FIXME: might race with other clones
if wasleaf:
self.setLeaf()
self.teardown(self.sdUUID, self.volUUID)
raise se.CannotCloneVolume(self.volumePath, dstPath, str(e))
def rebaseVolumeRollback(cls, taskObj, sdUUID, srcImg,
srcVol, dstFormat, srcParent, unsafe):
"""
Rebase volume rollback
"""
cls.log.info('rebase volume rollback (sdUUID=%s srcImg=%s srcVol=%s '
'dstFormat=%s srcParent=%s)', sdUUID, srcImg, srcVol,
dstFormat, srcParent)
imageResourcesNamespace = sd.getNamespace(sc.IMAGE_NAMESPACE, sdUUID)
with rm.acquireResource(imageResourcesNamespace, srcImg, rm.EXCLUSIVE):
vol = sdCache.produce(sdUUID).produceVolume(srcImg, srcVol)
vol.prepare(rw=True, chainrw=True, setrw=True)
volumePath = vol.getVolumePath()
backingVolPath = getBackingVolumePath(srcImg, srcParent)
try:
qemuimg.rebase(volumePath, backingVolPath,
sc.fmt2str(vol.getFormat()),
sc.fmt2str(int(dstFormat)),
misc.parseBool(unsafe), vars.task.aborting)
vol.setParent(srcParent)
vol.recheckIfLeaf()
except qemuimg.QImgError:
cls.log.exception('cannot rollback rebase for volume %s on '
'%s', volumePath, backingVolPath)
raise se.MergeVolumeRollbackError(srcVol)
finally:
vol.teardown(sdUUID, srcVol)
def test_chain_after_finalize(self, base_fmt):
with self.make_env(format=base_fmt, chain_len=3) as env:
base_vol = env.chain[0]
# We write data to the base and will read it from the child volume
# to verify that the chain is valid after qemu-rebase.
offset = 0
pattern = 0xf0
length = 1024
qemuio.write_pattern(
base_vol.volumePath,
sc.fmt2str(base_vol.getFormat()),
offset=offset,
len=length,
pattern=pattern)
top_vol = env.chain[1]
child_vol = env.chain[2]
subchain_info = dict(sd_id=base_vol.sdUUID,
img_id=base_vol.imgUUID,
base_id=base_vol.volUUID,
top_id=top_vol.volUUID,
base_generation=0)
subchain = merge.SubchainInfo(subchain_info, 0)
merge.finalize(subchain)
qemuio.verify_pattern(
child_vol.volumePath,
sc.fmt2str(child_vol.getFormat()),
offset=offset,
len=length,
pattern=pattern)
def verify_qemu_chain(vol_list):
# Check the integrity of a volume chain by reading the leaf volume
# and verifying the pattern written by write_chain. Also, check each
# volume in the chain to ensure it contains the correct data.
top_vol = vol_list[-1]
top_vol_fmt = sc.fmt2str(top_vol.getFormat())
for i, vol in enumerate(vol_list):
offset = i * 1024
pattern = 0xf0 + i
# Check that the correct pattern can be read through the top volume
qemu_pattern_verify(top_vol.volumePath,
top_vol_fmt,
offset=offset,
len=1024,
pattern=pattern)
# Check the volume where the pattern was originally written
vol_fmt = sc.fmt2str(vol.getFormat())
qemu_pattern_verify(vol.volumePath,
vol_fmt,
offset=offset,
len=1024,
pattern=pattern)
# Check that the next offset contains zeroes. If we know this layer
# has zeroes at next_offset we can be sure that data read at the same
# offset in the next layer belongs to that layer.
next_offset = (i + 1) * 1024
qemu_pattern_verify(vol.volumePath,
vol_fmt,
offset=next_offset,
len=1024,
pattern=0)
def rebase(self, backingVol, backingVolPath, backingFormat, unsafe,
rollback):
"""
Rebase volume on top of new backing volume
"""
if rollback:
pvol = self.getParentVolume()
if not pvol:
self.log.warn("Can't rebase volume %s, parent missing",
self.volUUID)
return
name = "Merge volume: " + self.volUUID
vars.task.pushRecovery(
task.Recovery(name, "volume", "Volume",
"rebaseVolumeRollback",
[self.sdUUID, self.getImage(),
self.volUUID, str(pvol.getFormat()),
pvol.volUUID, str(True)]))
volumePath = self.getVolumePath()
try:
qemuimg.rebase(volumePath, backingVolPath,
sc.fmt2str(self.getFormat()),
sc.fmt2str(backingFormat), unsafe,
vars.task.aborting)
except qemuimg.QImgError:
self.log.exception('cannot rebase volume %s on %s', volumePath,
backingVolPath)
raise se.MergeSnapshotsError(self.volUUID)
self.setParent(backingVol)
self.recheckIfLeaf()
def verify_qemu_chain(vol_list):
# Check the integrity of a volume chain by reading the leaf volume
# and verifying the pattern written by write_chain. Also, check each
# volume in the chain to ensure it contains the correct data.
top_vol = vol_list[-1]
top_vol_fmt = sc.fmt2str(top_vol.getFormat())
for i, vol in enumerate(vol_list):
offset = i * 1024
pattern = 0xf0 + i
# Check that the correct pattern can be read through the top volume
qemu_pattern_verify(top_vol.volumePath, top_vol_fmt, offset=offset,
len=1024, pattern=pattern)
# Check the volume where the pattern was originally written
vol_fmt = sc.fmt2str(vol.getFormat())
qemu_pattern_verify(vol.volumePath, vol_fmt, offset=offset, len=1024,
pattern=pattern)
# Check that the next offset contains zeroes. If we know this layer
# has zeroes at next_offset we can be sure that data read at the same
# offset in the next layer belongs to that layer.
next_offset = (i + 1) * 1024
qemu_pattern_verify(vol.volumePath, vol_fmt, offset=next_offset,
len=1024, pattern=0)
def clone(self, dstPath, volFormat):
"""
Clone self volume to the specified dst_image_dir/dst_volUUID
"""
wasleaf = False
taskName = "parent volume rollback: " + self.volUUID
vars.task.pushRecovery(
task.Recovery(taskName, "volume", "Volume", "parentVolumeRollback",
[self.sdUUID, self.imgUUID, self.volUUID]))
if self.isLeaf():
wasleaf = True
self.setInternal()
try:
self.prepare(rw=False)
self.log.debug('cloning volume %s to %s', self.volumePath, dstPath)
parent = getBackingVolumePath(self.imgUUID, self.volUUID)
domain = sdCache.produce(self.sdUUID)
qemuimg.create(dstPath,
backing=parent,
format=sc.fmt2str(volFormat),
qcow2Compat=domain.qcow2_compat(),
backingFormat=sc.fmt2str(self.getFormat()))
self.teardown(self.sdUUID, self.volUUID)
except Exception as e:
self.log.exception('cannot clone image %s volume %s to %s',
self.imgUUID, self.volUUID, dstPath)
# FIXME: might race with other clones
if wasleaf:
self.setLeaf()
self.teardown(self.sdUUID, self.volUUID)
raise se.CannotCloneVolume(self.volumePath, dstPath, str(e))
def sparsify(self, tmpSdUUID, tmpImgUUID, tmpVolUUID, dstSdUUID,
dstImgUUID, dstVolUUID):
"""
Reduce sparse image size by converting free space on image to free
space on storage domain using virt-sparsify.
"""
self.log.info("tmpSdUUID=%s, tmpImgUUID=%s, tmpVolUUID=%s, "
"dstSdUUID=%s, dstImgUUID=%s, dstVolUUID=%s", tmpSdUUID,
tmpImgUUID, tmpVolUUID, dstSdUUID, dstImgUUID,
dstVolUUID)
tmpVolume = self._getSparsifyVolume(tmpSdUUID, tmpImgUUID, tmpVolUUID)
dstVolume = self._getSparsifyVolume(dstSdUUID, dstImgUUID, dstVolUUID)
if not dstVolume.isSparse():
raise se.VolumeNotSparse()
srcVolume = self._getSparsifyVolume(tmpSdUUID, tmpImgUUID,
tmpVolume.getParent())
tmpVolume.prepare()
try:
dstVolume.prepare()
try:
# By definition "sparsification" is implemented writing a file
# with zeroes as large as the entire file-system. So at least
# tmpVolume needs to be as large as the virtual disk size for
# the worst case.
# TODO: Some extra space may be needed for QCOW2 headers
tmpVolume.extend(tmpVolume.getCapacity())
# For the dstVolume we may think of an optimization where the
# extension is as large as the source (and at the end we
# shrinkToOptimalSize).
# TODO: Extend the dstVolume only as much as the actual size of
# srcVolume
# TODO: Some extra space may be needed for QCOW2 headers
dstVolume.extend(tmpVolume.getCapacity())
srcFormat = sc.fmt2str(srcVolume.getFormat())
dstFormat = sc.fmt2str(dstVolume.getFormat())
virtsparsify.sparsify(srcVolume.getVolumePath(),
tmpVolume.getVolumePath(),
dstVolume.getVolumePath(),
src_format=srcFormat,
dst_format=dstFormat)
except Exception:
self.log.exception('Unexpected error sparsifying %s',
tmpVolUUID)
raise se.CannotSparsifyVolume(tmpVolUUID)
finally:
dstVolume.teardown(sdUUID=dstSdUUID, volUUID=dstVolUUID)
finally:
tmpVolume.teardown(sdUUID=tmpSdUUID, volUUID=tmpVolUUID)
self._shrinkVolumeToOptimalSize(tmpVolume)
self._shrinkVolumeToOptimalSize(dstVolume)
def test_copy_data_collapse(tmpdir, tmp_repo, fake_access, fake_rescan, tmp_db,
fake_task, fake_scheduler, monkeypatch,
dest_format, sd_version):
dom = tmp_repo.create_localfs_domain(name="domain", version=sd_version)
chain_size = 3
volumes = create_chain(dom, chain_size)
dest_img_id = str(uuid.uuid4())
dest_vol_id = str(uuid.uuid4())
length = MiB
# Write some data to each layer
for i, vol in enumerate(volumes):
qemuio.write_pattern(vol.getVolumePath(),
sc.fmt2str(vol.getFormat()),
offset=(i * length))
# The last volume in the chain is the leaf
source_leaf_vol = volumes[-1]
dest_vol = create_volume(dom,
dest_img_id,
dest_vol_id,
volFormat=dest_format)
source = dict(endpoint_type='div',
sd_id=source_leaf_vol.sdUUID,
img_id=source_leaf_vol.imgUUID,
vol_id=source_leaf_vol.volUUID)
dest = dict(endpoint_type='div',
sd_id=source_leaf_vol.sdUUID,
img_id=dest_img_id,
vol_id=dest_vol_id)
# Run copy_data from the source chain to dest_vol, essentially
# executing qemu-img convert
job = copy_data.Job(str(uuid.uuid4()), 0, source, dest)
monkeypatch.setattr(guarded, 'context', fake_guarded_context())
job.run()
# Source chain and destination image must have the same data but allocation
# may differ.
op = qemuimg.compare(source_leaf_vol.getVolumePath(),
dest_vol.getVolumePath(),
img1_format='qcow2',
img2_format=sc.fmt2str(dest_format),
strict=False)
op.run()
# Destination actual size should be smaller than source chain actual size,
# since we have only one qcow2 header (qcow2), or no header (raw).
src_actual_size = sum(
qemuimg.info(vol.getVolumePath())["actualsize"] for vol in volumes)
dst_actual_size = qemuimg.info(dest_vol.getVolumePath())["actualsize"]
assert dst_actual_size < src_actual_size
def estimate_qcow2_size(self, src_vol_params, dst_sd_id):
"""
Calculate volume allocation size for converting raw/qcow2
source volume to qcow2 volume on destination storage domain.
Arguments:
src_vol_params(dict): Dictionary returned from
`storage.volume.Volume.getVolumeParams()`
dst_sd_id(str) : Destination volume storage domain id
Returns:
Volume allocation in bytes
"""
# measure required size.
qemu_measure = qemuimg.measure(image=src_vol_params['path'],
format=sc.fmt2str(
src_vol_params['volFormat']),
output_format=qemuimg.FORMAT.QCOW2)
# Adds extra room so we don't have to extend this disk immediately
# when a vm is started.
chunk_size_mb = config.getint("irs", "volume_utilization_chunk_mb")
chunk_size = chunk_size_mb * MiB
required = (qemu_measure["required"] + chunk_size)
# Limit estimates size by maximum size.
vol_class = sdCache.produce(dst_sd_id).getVolumeClass()
max_size = vol_class.max_size(src_vol_params['capacity'],
sc.COW_FORMAT)
allocation = min(required, max_size)
# Return estimated size of allocation.
self.log.debug("Estimated allocation for qcow2 volume:"
"%d", allocation)
return allocation
def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt):
src_fmt = sc.name2type(src_fmt)
dst_fmt = sc.name2type(dst_fmt)
job_id = make_uuid()
with self.make_env(env_type, src_fmt, dst_fmt) as env:
src_vol = env.src_chain[0]
dst_vol = env.dst_chain[0]
write_qemu_chain(env.src_chain)
self.assertRaises(ChainVerificationError,
verify_qemu_chain, env.dst_chain)
source = dict(endpoint_type='div', sd_id=src_vol.sdUUID,
img_id=src_vol.imgUUID, vol_id=src_vol.volUUID)
dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID,
img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID)
job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest)
job.run()
wait_for_job(job)
self.assertEqual(sorted(self.expected_locks(src_vol, dst_vol)),
sorted(guarded.context.locks))
self.assertEqual(jobs.STATUS.DONE, job.status)
self.assertEqual(100.0, job.progress)
self.assertNotIn('error', job.info())
verify_qemu_chain(env.dst_chain)
self.assertEqual(sc.fmt2str(dst_fmt),
qemuimg.info(dst_vol.volumePath)['format'])
def _run(self):
self.log.info("Merging subchain %s", self.subchain)
with guarded.context(self.subchain.locks):
self.subchain.validate()
with self.subchain.prepare(), self.subchain.volume_operation():
top_vol_path = self.subchain.top_vol.getVolumePath()
base_vol_path = self.subchain.base_vol.getVolumePath()
self.log.info("Committing data from %s to %s", top_vol_path,
base_vol_path)
self.operation = qemuimg.commit(
top_vol_path,
topFormat=sc.fmt2str(self.subchain.top_vol.getFormat()),
base=base_vol_path)
self.operation.run()
if (self.subchain.base_vol.getFormat() == sc.COW_FORMAT
and self.merge_bitmaps):
self.log.info("Merging bitmaps from %s to %s",
top_vol_path, base_vol_path)
# Add and merge all the bitmaps from top_vol that don't
# exist on the base_vol and not handled by block-commit.
base_parent_vol = self.subchain.base_vol.getParentVolume()
base_parent_path = (base_parent_vol.getVolumePath()
if base_parent_vol else None)
bitmaps.merge_bitmaps(base_vol_path,
top_vol_path,
base_parent_path=base_parent_path)
def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt):
src_fmt = sc.name2type(src_fmt)
dst_fmt = sc.name2type(dst_fmt)
job_id = make_uuid()
with self.get_vols(env_type, src_fmt,
dst_fmt) as (src_chain, dst_chain):
src_vol = src_chain[0]
dst_vol = dst_chain[0]
write_qemu_chain(src_chain)
self.assertRaises(ChainVerificationError, verify_qemu_chain,
dst_chain)
source = dict(endpoint_type='div',
sd_id=src_vol.sdUUID,
img_id=src_vol.imgUUID,
vol_id=src_vol.volUUID)
dest = dict(endpoint_type='div',
sd_id=dst_vol.sdUUID,
img_id=dst_vol.imgUUID,
vol_id=dst_vol.volUUID)
job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest)
job.run()
wait_for_job(job)
self.assertEqual(sorted(self.expected_locks(src_vol, dst_vol)),
sorted(guarded.context.locks))
self.assertEqual(jobs.STATUS.DONE, job.status)
self.assertEqual(100.0, job.progress)
self.assertNotIn('error', job.info())
verify_qemu_chain(dst_chain)
self.assertEqual(sc.fmt2str(dst_fmt),
qemuimg.info(dst_vol.volumePath)['format'])
def test_intra_domain_copy(env_type, src_fmt, dst_fmt):
src_fmt = sc.name2type(src_fmt)
dst_fmt = sc.name2type(dst_fmt)
job_id = make_uuid()
with make_env(env_type, src_fmt, dst_fmt) as env:
src_vol = env.src_chain[0]
dst_vol = env.dst_chain[0]
write_qemu_chain(env.src_chain)
with pytest.raises(qemuio.VerificationError):
verify_qemu_chain(env.dst_chain)
source = dict(endpoint_type='div',
sd_id=src_vol.sdUUID,
img_id=src_vol.imgUUID,
vol_id=src_vol.volUUID)
dest = dict(endpoint_type='div',
sd_id=dst_vol.sdUUID,
img_id=dst_vol.imgUUID,
vol_id=dst_vol.volUUID)
job = copy_data.Job(job_id, 0, source, dest)
job.run()
assert (sorted(expected_locks(src_vol, dst_vol)) == sorted(
guarded.context.locks))
assert jobs.STATUS.DONE == job.status
assert 100.0 == job.progress
assert 'error' not in job.info()
verify_qemu_chain(env.dst_chain)
assert (sc.fmt2str(dst_fmt) == qemuimg.info(
dst_vol.volumePath)['format'])
def _create_cow_volume(
cls, dom, vol_id, capacity, vol_path, initial_size, vol_parent,
img_id, src_img_id, src_vol_id):
"""
specific implementation of _create() for COW volumes.
All the exceptions are properly handled and logged in volume.create()
"""
if initial_size:
cls.log.error("initial size is not supported "
"for file-based volumes")
raise se.InvalidParameterException("initial size", initial_size)
cls._truncate_volume(vol_path, 0, vol_id, dom)
if not vol_parent:
cls.log.info("Request to create COW volume %s with capacity = %s",
vol_path, capacity)
operation = qemuimg.create(vol_path,
size=capacity,
format=sc.fmt2str(sc.COW_FORMAT),
qcow2Compat=dom.qcow2_compat())
operation.run()
else:
# Create hardlink to template and its meta file
cls.log.info("Request to create snapshot %s/%s of volume %s/%s "
"with capacity %s",
img_id, vol_id, src_img_id, src_vol_id, capacity)
vol_parent.clone(vol_path, sc.COW_FORMAT, capacity)
# Forcing the volume permissions in case one of the tools we use
# (dd, qemu-img, etc.) will mistakenly change the file permissions.
cls._set_permissions(vol_path, dom)
return (vol_path,)
def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt):
src_fmt = sc.name2type(src_fmt)
dst_fmt = sc.name2type(dst_fmt)
job_id = str(uuid.uuid4())
with self.get_vols(env_type, src_fmt, dst_fmt) as (src_chain,
dst_chain):
src_vol = src_chain[0]
dst_vol = dst_chain[0]
write_qemu_chain(src_chain)
self.assertRaises(ChainVerificationError,
verify_qemu_chain, dst_chain)
source = dict(endpoint_type='div', sd_id=src_vol.sdUUID,
img_id=src_vol.imgUUID, vol_id=src_vol.volUUID)
dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID,
img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID)
job = storage.sdm.api.copy_data.Job(job_id, None, source, dest)
job.run()
wait_for_job(job)
self.assertEqual(jobs.STATUS.DONE, job.status)
self.assertEqual(100.0, job.progress)
self.assertNotIn('error', job.info())
verify_qemu_chain(dst_chain)
self.assertEqual(sc.fmt2str(dst_fmt),
qemuimg.info(dst_vol.volumePath)['format'])
def test_optimal_size_cow_leaf_not_empty(self):
# verify that optimal size is limited to max size.
with self.make_volume(size=GIB, format=sc.COW_FORMAT) as vol:
qemu_pattern_write(path=vol.volumePath,
format=sc.fmt2str(vol.getFormat()),
len=200 * MEGAB)
max_size = vol.max_size(GIB, vol.getFormat())
self.assertEqual(vol.optimal_size(), max_size)
def test_optimal_size_cow_leaf_not_empty(self):
# verify that optimal size is limited to max size.
with self.make_volume(size=GIB, format=sc.COW_FORMAT) as vol:
qemu_pattern_write(path=vol.volumePath,
format=sc.fmt2str(vol.getFormat()),
len=200 * MEGAB)
max_size = vol.max_size(GIB, vol.getFormat())
self.assertEqual(vol.optimal_size(), max_size)
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
volParent, srcImgUUID, srcVolUUID, volPath,
initialSize=None):
"""
Class specific implementation of volumeCreate. All the exceptions are
properly handled and logged in volume.create()
"""
if initialSize:
cls.log.error("initialSize is not supported for file-based "
"volumes")
raise se.InvalidParameterException("initial size",
initialSize)
sizeBytes = size * BLOCK_SIZE
truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0
try:
oop.getProcessPool(dom.sdUUID).truncateFile(
volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS,
creatExcl=True)
except OSError as e:
if e.errno == errno.EEXIST:
raise se.VolumeAlreadyExists(volUUID)
raise
if preallocate == sc.PREALLOCATED_VOL:
try:
operation = fallocate.allocate(volPath,
sizeBytes)
with vars.task.abort_callback(operation.abort):
with utils.stopwatch("Preallocating volume %s" % volPath):
operation.run()
except exception.ActionStopped:
raise
except Exception:
cls.log.error("Unexpected error", exc_info=True)
raise se.VolumesZeroingError(volPath)
if not volParent:
cls.log.info("Request to create %s volume %s with size = %s "
"sectors", sc.type2name(volFormat), volPath,
size)
if volFormat == sc.COW_FORMAT:
qemuimg.create(volPath,
size=sizeBytes,
format=sc.fmt2str(volFormat),
qcow2Compat=dom.qcow2_compat())
else:
# Create hardlink to template and its meta file
cls.log.info("Request to create snapshot %s/%s of volume %s/%s",
imgUUID, volUUID, srcImgUUID, srcVolUUID)
volParent.clone(volPath, volFormat)
# Forcing the volume permissions in case one of the tools we use
# (dd, qemu-img, etc.) will mistakenly change the file permissiosn.
dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS)
return (volPath,)
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
volParent, srcImgUUID, srcVolUUID, volPath,
initialSize=None):
"""
Class specific implementation of volumeCreate. All the exceptions are
properly handled and logged in volume.create()
"""
if initialSize:
cls.log.error("initialSize is not supported for file-based "
"volumes")
raise se.InvalidParameterException("initial size",
initialSize)
sizeBytes = size * BLOCK_SIZE
truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0
try:
oop.getProcessPool(dom.sdUUID).truncateFile(
volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS,
creatExcl=True)
except OSError as e:
if e.errno == errno.EEXIST:
raise se.VolumeAlreadyExists(volUUID)
raise
if preallocate == sc.PREALLOCATED_VOL:
try:
operation = fallocate.allocate(volPath,
sizeBytes)
with vars.task.abort_callback(operation.abort):
with utils.stopwatch("Preallocating volume %s" % volPath):
operation.run()
except exception.ActionStopped:
raise
except Exception:
cls.log.error("Unexpected error", exc_info=True)
raise se.VolumesZeroingError(volPath)
if not volParent:
cls.log.info("Request to create %s volume %s with size = %s "
"sectors", sc.type2name(volFormat), volPath,
size)
if volFormat == sc.COW_FORMAT:
qemuimg.create(volPath,
size=sizeBytes,
format=sc.fmt2str(volFormat),
qcow2Compat=dom.qcow2_compat())
else:
# Create hardlink to template and its meta file
cls.log.info("Request to create snapshot %s/%s of volume %s/%s",
imgUUID, volUUID, srcImgUUID, srcVolUUID)
volParent.clone(volPath, volFormat)
# Forcing the volume permissions in case one of the tools we use
# (dd, qemu-img, etc.) will mistakenly change the file permissiosn.
dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS)
return (volPath,)
def _rebase_operation(base, child):
backing = volume.getBackingVolumePath(base.imgUUID, base.volUUID)
backing_format = sc.fmt2str(base.getFormat())
operation = qemuimg.rebase(image=child.volumePath,
backing=backing,
format=qemuimg.FORMAT.QCOW2,
backingFormat=backing_format,
unsafe=True)
return operation
def _rebase_operation(base, child):
backing = volume.getBackingVolumePath(base.imgUUID, base.volUUID)
backing_format = sc.fmt2str(base.getFormat())
operation = qemuimg.rebase(image=child.volumePath,
backing=backing,
format=qemuimg.FORMAT.QCOW2,
backingFormat=backing_format,
unsafe=True)
return operation
def test_copy_data_collapse(tmpdir, tmp_repo, fake_access, fake_rescan, tmp_db,
fake_task, fake_scheduler, monkeypatch,
dest_format):
dom = tmp_repo.create_localfs_domain(name="domain", version=5)
chain_size = 3
volumes = create_chain(dom, chain_size)
dest_img_id = str(uuid.uuid4())
dest_vol_id = str(uuid.uuid4())
length = MEGAB
# Write some data to each layer
for i, vol in enumerate(volumes):
qemuio.write_pattern(vol.getVolumePath(),
sc.fmt2str(vol.getFormat()),
offset=(i * length))
# The last volume in the chain is the leaf
source_leaf_vol = volumes[-1]
dest_vol = create_volume(dom, dest_img_id, dest_vol_id, dest_format)
source = dict(endpoint_type='div',
sd_id=source_leaf_vol.sdUUID,
img_id=source_leaf_vol.imgUUID,
vol_id=source_leaf_vol.volUUID)
dest = dict(endpoint_type='div',
sd_id=source_leaf_vol.sdUUID,
img_id=dest_img_id,
vol_id=dest_vol_id)
# Run copy_data from the source chain to dest_vol, essentially
# executing qemu-img convert
job = copy_data.Job(str(uuid.uuid4()), 0, source, dest)
monkeypatch.setattr(guarded, 'context', fake_guarded_context())
job.run()
# verify the data written to the source chain is available on the
# collapsed target volume
for i in range(chain_size):
qemuio.verify_pattern(dest_vol.getVolumePath(),
sc.fmt2str(dest_vol.getFormat()),
offset=(i * length))
def _run(self):
self.log.info("Merging subchain %s", self.subchain)
with guarded.context(self.subchain.locks):
self.subchain.validate()
with self.subchain.prepare(), self.subchain.volume_operation():
self.operation = qemuimg.commit(
self.subchain.top_vol.getVolumePath(),
topFormat=sc.fmt2str(self.subchain.top_vol.getFormat()),
base=self.subchain.base_vol.getVolumePath())
self.operation.run()
def _run(self):
self.log.info("Merging subchain %s", self.subchain)
with guarded.context(self.subchain.locks):
self.subchain.validate()
with self.subchain.prepare(), self.subchain.volume_operation():
self.operation = qemuimg.commit(
self.subchain.top_vol.getVolumePath(),
topFormat=sc.fmt2str(self.subchain.top_vol.getFormat()),
base=self.subchain.base_vol.getVolumePath())
self.operation.run()
def test_ok(self, vol_fmt):
with self.fake_volume(vol_fmt) as vol:
qemu_fmt = sc.fmt2str(vol_fmt)
op = qemuimg.create(vol.volumePath,
size=self.SIZE,
format=qemu_fmt)
op.run()
h = FakeHSM()
h.verify_untrusted_volume('sp', vol.sdUUID, vol.imgUUID,
vol.volUUID)
def updateInvalidatedSize(self):
# During some complex flows the volume size might have been marked as
# invalidated (e.g. during a transaction). Here we are checking
# NOTE: the prerequisite to run this is that the volume is accessible
# (e.g. lv active) and not in use by another process (e.g. dd, qemu).
# Going directly to the metadata parameter as we should skip the size
# validation in getSize.
if int(self.getMetaParam(sc.SIZE)) < 1:
volInfo = qemuimg.info(self.getVolumePath(),
sc.fmt2str(self.getFormat()))
# qemu/qemu-img rounds down
self.setSize(volInfo['virtualsize'] / sc.BLOCK_SIZE)
def _create(cls, dom, imgUUID, volUUID, capacity, volFormat, preallocate,
volParent, srcImgUUID, srcVolUUID, volPath, initial_size=None,
add_bitmaps=False):
"""
Class specific implementation of volumeCreate. All the exceptions are
properly handled and logged in volume.create()
"""
lv_size = cls.calculate_volume_alloc_size(
preallocate, volFormat, capacity, initial_size)
lv_size_mb = utils.round(lv_size, MiB) // MiB
lvm.createLV(dom.sdUUID, volUUID, lv_size_mb, activate=True,
initialTags=(sc.TAG_VOL_UNINIT,))
fileutils.rm_file(volPath)
lvPath = lvm.lvPath(dom.sdUUID, volUUID)
cls.log.info("Creating volume symlink from %r to %r", lvPath, volPath)
os.symlink(lvPath, volPath)
if not volParent:
cls.log.info("Request to create %s volume %s with capacity = %s",
sc.type2name(volFormat), volPath, capacity)
if volFormat == sc.COW_FORMAT:
operation = qemuimg.create(volPath,
size=capacity,
format=sc.fmt2str(volFormat),
qcow2Compat=dom.qcow2_compat())
operation.run()
else:
# Create hardlink to template and its meta file
cls.log.info("Request to create snapshot %s/%s of volume %s/%s "
"with capacity %s",
imgUUID, volUUID, srcImgUUID, srcVolUUID, capacity)
volParent.clone(volPath, volFormat, capacity)
with dom.acquireVolumeMetadataSlot(volUUID) as slot:
mdTags = ["%s%s" % (sc.TAG_PREFIX_MD, slot),
"%s%s" % (sc.TAG_PREFIX_PARENT, srcVolUUID),
"%s%s" % (sc.TAG_PREFIX_IMAGE, imgUUID)]
lvm.changeLVsTags(
dom.sdUUID,
(volUUID,),
delTags=[sc.TAG_VOL_UNINIT],
addTags=mdTags)
try:
lvm.deactivateLVs(dom.sdUUID, [volUUID])
except se.CannotDeactivateLogicalVolume:
cls.log.warn("Cannot deactivate new created volume %s/%s",
dom.sdUUID, volUUID, exc_info=True)
return (dom.sdUUID, slot)
def updateInvalidatedSize(self):
# During some complex flows the volume size might have been marked as
# invalidated (e.g. during a transaction). Here we are checking
# NOTE: the prerequisite to run this is that the volume is accessible
# (e.g. lv active) and not in use by another process (e.g. dd, qemu).
# Going directly to the metadata parameter as we should skip the size
# validation in getSize.
if int(self.getMetaParam(sc.SIZE)) < 1:
volInfo = qemuimg.info(
self.getVolumePath(), sc.fmt2str(self.getFormat()))
# qemu/qemu-img rounds down
self.setSize(volInfo['virtualsize'] / sc.BLOCK_SIZE)
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None):
"""
Class specific implementation of volumeCreate. All the exceptions are
properly handled and logged in volume.create()
"""
lvSize = cls.calculate_volume_alloc_size(preallocate,
size, initialSize)
lvm.createLV(dom.sdUUID, volUUID, lvSize, activate=True,
initialTags=(sc.TAG_VOL_UNINIT,))
fileutils.rm_file(volPath)
lvPath = lvm.lvPath(dom.sdUUID, volUUID)
cls.log.info("Creating volume symlink from %r to %r", lvPath, volPath)
os.symlink(lvPath, volPath)
if not volParent:
cls.log.info("Request to create %s volume %s with size = %s "
"blocks", sc.type2name(volFormat), volPath,
size)
if volFormat == sc.COW_FORMAT:
operation = qemuimg.create(volPath,
size=size * BLOCK_SIZE,
format=sc.fmt2str(volFormat),
qcow2Compat=dom.qcow2_compat())
operation.run()
else:
# Create hardlink to template and its meta file
cls.log.info("Request to create snapshot %s/%s of volume %s/%s "
"with size %s (blocks)",
imgUUID, volUUID, srcImgUUID, srcVolUUID, size)
volParent.clone(volPath, volFormat, size)
with dom.acquireVolumeMetadataSlot(volUUID) as slot:
mdTags = ["%s%s" % (sc.TAG_PREFIX_MD, slot),
"%s%s" % (sc.TAG_PREFIX_PARENT, srcVolUUID),
"%s%s" % (sc.TAG_PREFIX_IMAGE, imgUUID)]
lvm.changeLVTags(dom.sdUUID, volUUID, delTags=[sc.TAG_VOL_UNINIT],
addTags=mdTags)
try:
lvm.deactivateLVs(dom.sdUUID, [volUUID])
except se.CannotDeactivateLogicalVolume:
cls.log.warn("Cannot deactivate new created volume %s/%s",
dom.sdUUID, volUUID, exc_info=True)
return (dom.sdUUID, slot)
def rebase(self, backingVol, backingVolPath, backingFormat, unsafe,
rollback):
"""
Rebase volume on top of new backing volume
"""
if rollback:
pvol = self.getParentVolume()
if not pvol:
self.log.warn("Can't rebase volume %s, parent missing",
self.volUUID)
return
name = "Merge volume: " + self.volUUID
vars.task.pushRecovery(
task.Recovery(name, "volume", "Volume", "rebaseVolumeRollback",
[
self.sdUUID,
self.getImage(), self.volUUID,
str(pvol.getFormat()), pvol.volUUID,
str(True)
]))
volumePath = self.getVolumePath()
try:
qemuimg.rebase(volumePath, backingVolPath,
sc.fmt2str(self.getFormat()),
sc.fmt2str(backingFormat), unsafe,
vars.task.aborting)
except qemuimg.QImgError:
self.log.exception('cannot rebase volume %s on %s', volumePath,
backingVolPath)
raise se.MergeSnapshotsError(self.volUUID)
self.setParent(backingVol)
self.recheckIfLeaf()
def write_qemu_chain(vol_list):
# Starting with the base volume in vol_list, write to the chain in a
# pattern like the following:
#
# logical offset: 0K 1K 2K 3K
# Base Volume 0: 0xf0 0xf0 ...
# 1: 0xf1 0xf1 ...
# 2: 0xf2 0xf2 ...
# Leaf Volume 3: 0xf3 0xf3 ...
# This allows us to verify the integrity of the whole chain.
for i, vol in enumerate(vol_list):
vol_fmt = sc.fmt2str(vol.getFormat())
offset = i * 1024
pattern = 0xf0 + i
qemu_pattern_write(vol.volumePath, vol_fmt, offset=offset,
len=1024, pattern=pattern)
def write_qemu_chain(vol_list):
# Starting with the base volume in vol_list, write to the chain in a
# pattern like the following:
#
# logical offset: 0K 1K 2K 3K
# Base Volume 0: 0xf0 0xf0 ...
# 1: 0xf1 0xf1 ...
# 2: 0xf2 0xf2 ...
# Leaf Volume 3: 0xf3 0xf3 ...
# This allows us to verify the integrity of the whole chain.
for i, vol in enumerate(vol_list):
vol_fmt = sc.fmt2str(vol.getFormat())
offset = i * 1024
pattern = 0xf0 + i
qemu_pattern_write(vol.volumePath, vol_fmt, offset=offset,
len=1024, pattern=pattern)
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None):
"""
Class specific implementation of volumeCreate. All the exceptions are
properly handled and logged in volume.create()
"""
lvSize = cls.calculate_volume_alloc_size(preallocate,
size, initialSize)
lvm.createLV(dom.sdUUID, volUUID, "%s" % lvSize, activate=True,
initialTags=(sc.TAG_VOL_UNINIT,))
fileutils.rm_file(volPath)
os.symlink(lvm.lvPath(dom.sdUUID, volUUID), volPath)
if not volParent:
cls.log.info("Request to create %s volume %s with size = %s "
"sectors", sc.type2name(volFormat), volPath,
size)
if volFormat == sc.COW_FORMAT:
qemuimg.create(volPath,
size=size * BLOCK_SIZE,
format=sc.fmt2str(volFormat),
qcow2Compat=dom.qcow2_compat())
else:
# Create hardlink to template and its meta file
cls.log.info("Request to create snapshot %s/%s of volume %s/%s",
imgUUID, volUUID, srcImgUUID, srcVolUUID)
volParent.clone(volPath, volFormat)
with dom.acquireVolumeMetadataSlot(
volUUID, sc.VOLUME_MDNUMBLKS) as slot:
mdTags = ["%s%s" % (sc.TAG_PREFIX_MD, slot),
"%s%s" % (sc.TAG_PREFIX_PARENT, srcVolUUID),
"%s%s" % (sc.TAG_PREFIX_IMAGE, imgUUID)]
lvm.changeLVTags(dom.sdUUID, volUUID, delTags=[sc.TAG_VOL_UNINIT],
addTags=mdTags)
try:
lvm.deactivateLVs(dom.sdUUID, [volUUID])
except se.CannotDeactivateLogicalVolume:
cls.log.warn("Cannot deactivate new created volume %s/%s",
dom.sdUUID, volUUID, exc_info=True)
return (dom.sdUUID, slot)
def test_readonly(nbd_env, format, allocation):
# Volume served by qemu-nd.
img_id = str(uuid.uuid4())
vol_id = str(uuid.uuid4())
nbd_env.make_volume(nbd_env.virtual_size,
img_id,
vol_id,
vol_format=format,
prealloc=allocation)
# Fill volume with data before starting the server.
vol = nbd_env.sd_manifest.produceVolume(img_id, vol_id)
op = qemuimg.convert(nbd_env.src,
vol.getVolumePath(),
dstFormat=sc.fmt2str(format),
preallocation=PREALLOCATION.get(format))
op.run()
# Server configuration.
config = {
"sd_id": nbd_env.sd_manifest.sdUUID,
"img_id": img_id,
"vol_id": vol_id,
"readonly": True,
}
with nbd_server(config) as nbd_url:
# Writing to NBD server must fail.
with pytest.raises(cmdutils.Error):
op = qemuimg.convert(nbd_env.src,
nbd_url,
srcFormat="raw",
create=False)
op.run()
# Copy data from NBD server to dst. Both files should match byte
# for byte after the operation.
op = qemuimg.convert(nbd_url, nbd_env.dst, dstFormat="raw")
op.run()
with io.open(nbd_env.src) as s, io.open(nbd_env.dst) as d:
assert s.read() == d.read()
# Now the server should not be accessible.
with pytest.raises(cmdutils.Error):
qemuimg.info(nbd_url)
def test_readonly(nbd_env, format, allocation):
# Volume served by qemu-nd.
img_id = str(uuid.uuid4())
vol_id = str(uuid.uuid4())
nbd_env.make_volume(
nbd_env.virtual_size,
img_id,
vol_id,
vol_format=format,
prealloc=allocation)
# Fill volume with data before starting the server.
vol = nbd_env.sd_manifest.produceVolume(img_id, vol_id)
op = qemuimg.convert(
nbd_env.src,
vol.getVolumePath(),
dstFormat=sc.fmt2str(format),
preallocation=PREALLOCATION.get(format))
op.run()
# Server configuration.
config = {
"sd_id": nbd_env.sd_manifest.sdUUID,
"img_id": img_id,
"vol_id": vol_id,
"readonly": True,
}
with nbd_server(config) as nbd_url:
# Writing to NBD server must fail.
with pytest.raises(cmdutils.Error):
op = qemuimg.convert(
nbd_env.src, nbd_url, srcFormat="raw", create=False)
op.run()
# Copy data from NBD server to dst. Both files should match byte
# for byte after the operation.
op = qemuimg.convert(nbd_url, nbd_env.dst, dstFormat="raw")
op.run()
with io.open(nbd_env.src) as s, io.open(nbd_env.dst) as d:
assert s.read() == d.read()
# Now the server should not be accessible.
with pytest.raises(cmdutils.Error):
qemuimg.info(nbd_url)
def test_merge_subchain(self, sd_type, chain_len, base_index, top_index):
job_id = make_uuid()
with self.make_env(sd_type=sd_type, chain_len=chain_len) as env:
write_qemu_chain(env.chain)
base_vol = env.chain[base_index]
top_vol = env.chain[top_index]
subchain_info = dict(sd_id=base_vol.sdUUID,
img_id=base_vol.imgUUID,
base_id=base_vol.volUUID,
top_id=top_vol.volUUID,
base_generation=0)
subchain = merge.SubchainInfo(subchain_info, 0)
job = api_merge.Job(job_id, subchain)
job.run()
wait_for_job(job)
self.assertEqual(job.status, jobs.STATUS.DONE)
# Verify that the chain data was merged
for i in range(base_index, top_index + 1):
offset = i * 1024
pattern = 0xf0 + i
# We expect to read all data from top
verify_pattern(
top_vol.volumePath,
qemuimg.FORMAT.QCOW2,
offset=offset,
len=1024,
pattern=pattern)
# And base, since top was merged into base
verify_pattern(
base_vol.volumePath,
sc.fmt2str(base_vol.getFormat()),
offset=offset,
len=1024,
pattern=pattern)
self.assertEqual(sorted(self.expected_locks(base_vol)),
sorted(guarded.context.locks))
self.assertEqual(base_vol.getLegality(), sc.LEGAL_VOL)
self.assertEqual(base_vol.getMetaParam(sc.GENERATION), 1)
def _run(self):
self.log.info("Merging subchain %s", self.subchain)
with guarded.context(self.subchain.locks):
self.subchain.validate()
# Base volume must be ILLEGAL. Otherwise, VM could be run while
# performing cold merge.
base_legality = self.subchain.base_vol.getLegality()
if base_legality == sc.LEGAL_VOL:
raise se.UnexpectedVolumeState(self.subchain.base_id,
sc.ILLEGAL_VOL,
base_legality)
with self.subchain.prepare(), self.subchain.volume_operation():
self.operation = qemuimg.commit(
self.subchain.top_vol.getVolumePath(),
topFormat=sc.fmt2str(self.subchain.top_vol.getFormat()),
base=self.subchain.base_vol.getVolumePath())
with utils.closing(self.operation):
self.operation.wait_for_completion()
def reconcileVolumeChain(self, sdUUID, imgUUID, leafVolUUID):
"""
Discover and return the actual volume chain of an offline image
according to the qemu-img info command and synchronize volume metadata.
"""
# Prepare volumes
dom = sdCache.produce(sdUUID)
allVols = dom.getAllVolumes()
imgVolumes = sd.getVolsOfImage(allVols, imgUUID).keys()
dom.activateVolumes(imgUUID, imgVolumes)
# Walk the volume chain using qemu-img. Not safe for running VMs
actualVolumes = []
volUUID = leafVolUUID
while volUUID is not None:
actualVolumes.insert(0, volUUID)
vol = dom.produceVolume(imgUUID, volUUID)
qemuImgFormat = sc.fmt2str(vol.getFormat())
imgInfo = qemuimg.info(vol.volumePath, qemuImgFormat)
backingFile = imgInfo.get('backingfile')
if backingFile is not None:
volUUID = os.path.basename(backingFile)
else:
volUUID = None
# A merge of the active layer has copy and pivot phases.
# During copy, data is copied from the leaf into its parent. Writes
# are mirrored to both volumes. So even after copying is complete the
# volumes will remain consistent. Finally, the VM is pivoted from the
# old leaf to the new leaf and mirroring to the old leaf ceases. During
# mirroring and before pivoting, we mark the old leaf ILLEGAL so we
# know it's safe to delete in case the operation is interrupted.
vol = dom.produceVolume(imgUUID, leafVolUUID)
if vol.getLegality() == sc.ILLEGAL_VOL:
actualVolumes.remove(leafVolUUID)
# Now that we know the correct volume chain, sync the storge metadata
self.syncVolumeChain(sdUUID, imgUUID, actualVolumes[-1], actualVolumes)
dom.deactivateImage(imgUUID)
return actualVolumes
def _create_cow_volume(
cls, dom, vol_id, size, vol_path, initial_size, vol_parent,
img_id, src_img_id, src_vol_id):
"""
specific implementation of _create() for COW volumes.
All the exceptions are properly handled and logged in volume.create()
"""
if initial_size:
cls.log.error("initial size is not supported "
"for file-based volumes")
raise se.InvalidParameterException("initial size", initial_size)
cls._truncate_volume(vol_path, 0, vol_id, dom)
if not vol_parent:
cls.log.info("Request to create COW volume %s with size = %s "
"bytes", vol_path, size)
operation = qemuimg.create(vol_path,
size=size,
format=sc.fmt2str(sc.COW_FORMAT),
qcow2Compat=dom.qcow2_compat())
operation.run()
else:
# Create hardlink to template and its meta file
cls.log.info("Request to create snapshot %s/%s of volume %s/%s "
"with size %s (bytes)",
img_id, vol_id, src_img_id, src_vol_id, size)
size_blk = size // BLOCK_SIZE
vol_parent.clone(vol_path, sc.COW_FORMAT, size_blk)
# Forcing the volume permissions in case one of the tools we use
# (dd, qemu-img, etc.) will mistakenly change the file permissions.
cls._set_permissions(vol_path, dom)
return (vol_path,)
def qemu_format(self):
return sc.fmt2str(self.volume.getFormat())
def qemu_format(self):
# TODO: Use Image._detect_format to handle broken VM md images.
return sc.fmt2str(self._vol.getFormat())
def _interImagesCopy(self, destDom, srcSdUUID, imgUUID, chains):
srcLeafVol = chains['srcChain'][-1]
dstLeafVol = chains['dstChain'][-1]
try:
# Prepare the whole chains before the copy
srcLeafVol.prepare(rw=False)
dstLeafVol.prepare(rw=True, chainrw=True, setrw=True)
except Exception:
self.log.error("Unexpected error", exc_info=True)
# teardown volumes
self.__cleanupMove(srcLeafVol, dstLeafVol)
raise
try:
for srcVol in chains['srcChain']:
# Do the actual copy
try:
dstVol = destDom.produceVolume(imgUUID=imgUUID,
volUUID=srcVol.volUUID)
if workarounds.invalid_vm_conf_disk(srcVol):
srcFormat = dstFormat = qemuimg.FORMAT.RAW
else:
srcFormat = sc.fmt2str(srcVol.getFormat())
dstFormat = sc.fmt2str(dstVol.getFormat())
parentVol = dstVol.getParentVolume()
if parentVol is not None:
backing = volume.getBackingVolumePath(
imgUUID, parentVol.volUUID)
backingFormat = sc.fmt2str(parentVol.getFormat())
else:
backing = None
backingFormat = None
if (destDom.supportsSparseness
and dstVol.getType() == sc.PREALLOCATED_VOL):
preallocation = qemuimg.PREALLOCATION.FALLOC
else:
preallocation = None
operation = qemuimg.convert(
srcVol.getVolumePath(),
dstVol.getVolumePath(),
srcFormat=srcFormat,
dstFormat=dstFormat,
dstQcow2Compat=destDom.qcow2_compat(),
backing=backing,
backingFormat=backingFormat,
preallocation=preallocation,
unordered_writes=destDom.recommends_unordered_writes(
dstVol.getFormat()),
create=not destDom.is_block(),
)
with utils.stopwatch("Copy volume %s" % srcVol.volUUID):
self._run_qemuimg_operation(operation)
except ActionStopped:
raise
except se.StorageException:
self.log.error("Unexpected error", exc_info=True)
raise
except Exception:
self.log.error(
"Copy image error: image=%s, src domain=%s,"
" dst domain=%s",
imgUUID,
srcSdUUID,
destDom.sdUUID,
exc_info=True)
raise se.CopyImageError()
finally:
# teardown volumes
self.__cleanupMove(srcLeafVol, dstLeafVol)
def getQemuImageInfo(self):
"""
Returns volume information as returned by qemu-img info command
"""
return qemuimg.info(self.getVolumePath(), sc.fmt2str(self.getFormat()))
def _initialize_volume(self, vol_format, size):
if vol_format == sc.COW_FORMAT:
qemuimg.create(self.volume_path, size, sc.fmt2str(vol_format))
def copyCollapsed(self, sdUUID, vmUUID, srcImgUUID, srcVolUUID, dstImgUUID,
dstVolUUID, descr, dstSdUUID, volType, volFormat,
preallocate, postZero, force, discard):
"""
Create new template/volume from VM.
Do it by collapse and copy the whole chain (baseVolUUID->srcVolUUID)
"""
self.log.info(
"sdUUID=%s vmUUID=%s srcImgUUID=%s srcVolUUID=%s "
"dstImgUUID=%s dstVolUUID=%s dstSdUUID=%s volType=%s "
"volFormat=%s preallocate=%s force=%s postZero=%s "
"discard=%s", sdUUID, vmUUID, srcImgUUID, srcVolUUID, dstImgUUID,
dstVolUUID, dstSdUUID, volType, sc.type2name(volFormat),
sc.type2name(preallocate), str(force), str(postZero), discard)
try:
srcVol = dstVol = None
# Find out dest sdUUID
if dstSdUUID == sd.BLANK_UUID:
dstSdUUID = sdUUID
volclass = sdCache.produce(sdUUID).getVolumeClass()
destDom = sdCache.produce(dstSdUUID)
# find src volume
try:
srcVol = volclass(self.repoPath, sdUUID, srcImgUUID,
srcVolUUID)
except se.StorageException:
raise
except Exception as e:
self.log.error(e, exc_info=True)
raise se.SourceImageActionError(srcImgUUID, sdUUID, str(e))
# Create dst volume
try:
# Before reading source volume parameters from volume metadata,
# prepare the volume. This ensure that the volume capacity will
# match the actual virtual size, see
# https://bugzilla.redhat.com/1700623.
srcVol.prepare(rw=False)
volParams = srcVol.getVolumeParams()
if volFormat in [sc.COW_FORMAT, sc.RAW_FORMAT]:
dstVolFormat = volFormat
else:
dstVolFormat = volParams['volFormat']
# TODO: This is needed only when copying to qcow2-thin volume
# on block storage. Move into calculate_initial_size.
dst_vol_allocation = self.calculate_vol_alloc(
sdUUID, volParams, dstSdUUID, dstVolFormat)
# Find out dest volume parameters
if preallocate in [sc.PREALLOCATED_VOL, sc.SPARSE_VOL]:
volParams['prealloc'] = preallocate
initial_size = self.calculate_initial_size(
destDom.supportsSparseness, dstVolFormat,
volParams['prealloc'], dst_vol_allocation)
self.log.info(
"Copy source %s:%s:%s to destination %s:%s:%s "
"capacity=%s, initial size=%s", sdUUID, srcImgUUID,
srcVolUUID, dstSdUUID, dstImgUUID, dstVolUUID,
volParams['capacity'], initial_size)
# If image already exists check whether it illegal/fake,
# overwrite it
if not self.isLegal(dstSdUUID, dstImgUUID):
force = True
# We must first remove the previous instance of image (if
# exists) in destination domain, if we got the overwrite
# command
if force:
self.log.info(
"delete image %s on domain %s before "
"overwriting", dstImgUUID, dstSdUUID)
_deleteImage(destDom, dstImgUUID, postZero, discard)
destDom.createVolume(imgUUID=dstImgUUID,
capacity=volParams['capacity'],
volFormat=dstVolFormat,
preallocate=volParams['prealloc'],
diskType=volParams['disktype'],
volUUID=dstVolUUID,
desc=descr,
srcImgUUID=sc.BLANK_UUID,
srcVolUUID=sc.BLANK_UUID,
initial_size=initial_size)
dstVol = sdCache.produce(dstSdUUID).produceVolume(
imgUUID=dstImgUUID, volUUID=dstVolUUID)
except se.StorageException:
self.log.error("Unexpected error", exc_info=True)
raise
except Exception as e:
self.log.error("Unexpected error", exc_info=True)
raise se.CopyImageError("Destination volume %s error: %s" %
(dstVolUUID, str(e)))
try:
# Start the actual copy image procedure
dstVol.prepare(rw=True, setrw=True)
if (destDom.supportsSparseness
and dstVol.getType() == sc.PREALLOCATED_VOL):
preallocation = qemuimg.PREALLOCATION.FALLOC
else:
preallocation = None
try:
operation = qemuimg.convert(
volParams['path'],
dstVol.getVolumePath(),
srcFormat=sc.fmt2str(volParams['volFormat']),
dstFormat=sc.fmt2str(dstVolFormat),
dstQcow2Compat=destDom.qcow2_compat(),
preallocation=preallocation,
unordered_writes=destDom.recommends_unordered_writes(
dstVolFormat),
create=not destDom.is_block(),
)
with utils.stopwatch("Copy volume %s" % srcVol.volUUID):
self._run_qemuimg_operation(operation)
except ActionStopped:
raise
except cmdutils.Error as e:
self.log.exception('conversion failure for volume %s',
srcVol.volUUID)
raise se.CopyImageError(str(e))
# Mark volume as SHARED
if volType == sc.SHARED_VOL:
dstVol.setShared()
dstVol.setLegality(sc.LEGAL_VOL)
if force:
# Now we should re-link all deleted hardlinks, if exists
destDom.templateRelink(dstImgUUID, dstVolUUID)
except se.StorageException:
self.log.error("Unexpected error", exc_info=True)
raise
except Exception as e:
self.log.error("Unexpected error", exc_info=True)
raise se.CopyImageError("src image=%s, dst image=%s: msg=%s" %
(srcImgUUID, dstImgUUID, str(e)))
self.log.info("Finished copying %s:%s -> %s:%s", sdUUID,
srcVolUUID, dstSdUUID, dstVolUUID)
# TODO: handle return status
return dstVolUUID
finally:
self.__cleanupCopy(srcVol=srcVol, dstVol=dstVol)
def backing_qemu_format(self):
parent_vol = self._vol.getParentVolume()
if not parent_vol:
return None
return sc.fmt2str(parent_vol.getFormat())
def getQemuImageInfo(self):
"""
Returns volume information as returned by qemu-img info command
"""
return qemuimg.info(self.getVolumePath(),
sc.fmt2str(self.getFormat()))