def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2.
"""
LOG.debug("Snapshot: %(snap)s, volume: %(vol)s, "
"volume_size: %(size)s" %
{'snap': snapshot['id'],
'vol': volume['id'],
'size': volume_size})
info_path = self._local_path_volume_info(snapshot['volume'])
snap_info = self._read_info_file(info_path)
vol_dir = self._local_volume_dir(snapshot['volume'])
out_format = self.get_volume_format(volume, qemu_format=True)
forward_file = snap_info[snapshot['id']]
forward_path = os.path.join(vol_dir, forward_file)
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path,
snapshot['volume']['name'])
path_to_snap_img = os.path.join(vol_dir, img_info.backing_file)
LOG.debug("Will copy from snapshot at %s" % path_to_snap_img)
image_utils.convert_image(path_to_snap_img,
self.local_path(volume),
out_format)
self._extend_volume(volume, volume_size)
self._set_rw_permissions_for_all(self.local_path(volume))
def test_clone_image_cloneableshare_notraw(self):
drv = self._driver
mox = self.mox
volume = {"name": "vol", "size": "20"}
mox.StubOutWithMock(drv, "_find_image_in_cache")
mox.StubOutWithMock(drv, "_is_cloneable_share")
mox.StubOutWithMock(drv, "_get_mount_point_for_share")
mox.StubOutWithMock(image_utils, "qemu_img_info")
mox.StubOutWithMock(drv, "_clone_volume")
mox.StubOutWithMock(drv, "_discover_file_till_timeout")
mox.StubOutWithMock(drv, "_set_rw_permissions_for_all")
mox.StubOutWithMock(drv, "_resize_image_file")
mox.StubOutWithMock(image_utils, "convert_image")
mox.StubOutWithMock(drv, "_register_image_in_cache")
mox.StubOutWithMock(drv, "_is_share_vol_compatible")
drv._find_image_in_cache(IgnoreArg()).AndReturn([])
drv._is_cloneable_share("nfs://127.0.0.1/share/img-id").AndReturn("127.0.0.1:/share")
drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True)
drv._get_mount_point_for_share("127.0.0.1:/share").AndReturn("/mnt")
image_utils.qemu_img_info("/mnt/img-id").AndReturn(self.get_img_info("notraw"))
image_utils.convert_image(IgnoreArg(), IgnoreArg(), "raw")
image_utils.qemu_img_info("/mnt/vol").AndReturn(self.get_img_info("raw"))
drv._register_image_in_cache(IgnoreArg(), IgnoreArg())
drv._get_mount_point_for_share("127.0.0.1:/share").AndReturn("/mnt")
drv._discover_file_till_timeout(IgnoreArg()).AndReturn(True)
drv._set_rw_permissions_for_all("/mnt/vol")
drv._resize_image_file({"name": "vol"}, IgnoreArg())
mox.ReplayAll()
drv.clone_image(volume, ("nfs://127.0.0.1/share/img-id", None), "image_id")
mox.VerifyAll()
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
# If snapshots exist, flatten to a temporary image, and upload it
active_file = self.get_active_image_from_info(volume)
active_file_path = '%s/%s' % (self._local_volume_dir(volume),
active_file)
info = self._qemu_img_info(active_file_path)
backing_file = info.backing_file
if backing_file:
snapshots_exist = True
else:
snapshots_exist = False
root_file_fmt = info.file_format
tmp_params = {
'prefix': '%s.temp_image.%s' % (volume['id'], image_meta['id']),
'suffix': '.img'
}
with image_utils.temporary_file(**tmp_params) as temp_path:
if snapshots_exist or (root_file_fmt != 'raw'):
# Convert due to snapshots
# or volume data not being stored in raw format
# (upload_volume assumes raw format input)
image_utils.convert_image(active_file_path, temp_path, 'raw')
upload_path = temp_path
else:
upload_path = active_file_path
image_utils.upload_volume(context,
image_service,
image_meta,
upload_path)
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2.
"""
LOG.debug(
"snapshot: %(snap)s, volume: %(vol)s, ", {"snap": snapshot["id"], "vol": volume["id"], "size": volume_size}
)
info_path = self._local_path_volume_info(snapshot["volume"])
snap_info = self._read_info_file(info_path)
vol_path = self._local_volume_dir(snapshot["volume"])
forward_file = snap_info[snapshot["id"]]
forward_path = os.path.join(vol_path, forward_file)
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path, snapshot["volume"]["name"])
path_to_snap_img = os.path.join(vol_path, img_info.backing_file)
path_to_new_vol = self._local_path_volume(volume)
LOG.debug("will copy from snapshot at %s", path_to_snap_img)
if self.configuration.quobyte_qcow2_volumes:
out_format = "qcow2"
else:
out_format = "raw"
image_utils.convert_image(path_to_snap_img, path_to_new_vol, out_format, run_as_root=self._execute_as_root)
self._set_rw_permissions_for_all(path_to_new_vol)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
# If snapshots exist, flatten to a temporary image, and upload it
active_file = self.get_active_image_from_info(volume)
active_file_path = "%s/%s" % (self._local_volume_dir(volume), active_file)
backing_file = self._get_backing_file_for_path(active_file_path)
if backing_file is not None:
snapshots_exist = True
else:
snapshots_exist = False
root_file_fmt = self._get_file_format_for_path(self._local_path_volume(volume))
temp_path = None
try:
if snapshots_exist or (root_file_fmt != "raw"):
# Convert due to snapshots
# or volume data not being stored in raw format
# (upload_volume assumes raw format input)
temp_path = "%s/%s.temp_image.%s" % (self._local_volume_dir(volume), volume["id"], image_meta["id"])
image_utils.convert_image(active_file_path, temp_path, "raw")
upload_path = temp_path
else:
upload_path = active_file_path
image_utils.upload_volume(context, image_service, image_meta, upload_path)
finally:
if temp_path is not None:
self._execute("rm", "-f", temp_path)
def _direct_nfs_clone(self, volume, image_location, image_id):
"""Clone directly in nfs share."""
LOG.info(_("Cloning image %s directly in share"), image_id)
cloned = False
image_location = self._construct_image_nfs_url(image_location)
share = self._is_cloneable_share(image_location)
if share and self._is_share_eligible(share, volume["size"]):
LOG.debug(_("Share is cloneable %s"), share)
volume["provider_location"] = share
(__, ___, img_file) = image_location.rpartition("/")
dir_path = self._get_mount_point_for_share(share)
img_path = "%s/%s" % (dir_path, img_file)
img_info = image_utils.qemu_img_info(img_path)
if img_info.file_format == "raw":
LOG.debug(_("Image is raw %s"), image_id)
self._clone_volume(img_file, volume["name"], volume_id=None, share=share)
cloned = True
else:
LOG.info(_("Image will locally be converted to raw %s"), image_id)
dst = "%s/%s" % (dir_path, volume["name"])
image_utils.convert_image(img_path, dst, "raw")
data = image_utils.qemu_img_info(dst)
if data.file_format != "raw":
raise exception.InvalidResults(_("Converted to raw, but" " format is now %s") % data.file_format)
else:
cloned = True
self._register_image_in_cache(volume, image_id)
return cloned
def test_copy_volume_from_snapshot(self):
(mox, drv) = self._mox, self._driver
mox.StubOutWithMock(image_utils, 'convert_image')
mox.StubOutWithMock(drv, '_read_info_file')
mox.StubOutWithMock(image_utils, 'qemu_img_info')
mox.StubOutWithMock(drv, '_set_rw_permissions_for_all')
dest_volume = self._simple_volume(
'c1073000-0000-0000-0000-0000000c1073')
src_volume = self._simple_volume()
vol_dir = os.path.join(self.TEST_MNT_POINT_BASE,
drv._get_hash_str(self.TEST_QUOBYTE_VOLUME))
src_vol_path = os.path.join(vol_dir, src_volume['name'])
dest_vol_path = os.path.join(vol_dir, dest_volume['name'])
info_path = os.path.join(vol_dir, src_volume['name']) + '.info'
snapshot = {'volume_name': src_volume['name'],
'name': 'clone-snap-%s' % src_volume['id'],
'size': src_volume['size'],
'volume_size': src_volume['size'],
'volume_id': src_volume['id'],
'id': 'tmp-snap-%s' % src_volume['id'],
'volume': src_volume}
snap_file = dest_volume['name'] + '.' + snapshot['id']
snap_path = os.path.join(vol_dir, snap_file)
size = dest_volume['size']
drv._read_info_file(info_path).AndReturn(
{'active': snap_file,
snapshot['id']: snap_file}
)
qemu_img_output = """image: %s
file format: raw
virtual size: 1.0G (1073741824 bytes)
disk size: 173K
backing file: %s
""" % (snap_file, src_volume['name'])
img_info = imageutils.QemuImgInfo(qemu_img_output)
image_utils.qemu_img_info(snap_path).AndReturn(img_info)
image_utils.convert_image(src_vol_path,
dest_vol_path,
'raw',
run_as_root=self.execute_as_root)
drv._set_rw_permissions_for_all(dest_vol_path)
mox.ReplayAll()
drv._copy_volume_from_snapshot(snapshot, dest_volume, size)
mox.VerifyAll()
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2. If the quobyte_volume_from_snapshot_cache is active the result
is copied into the cache and all volumes created from this
snapshot id are directly copied from the cache.
"""
LOG.debug("snapshot: %(snap)s, volume: %(vol)s, ",
{'snap': snapshot.id,
'vol': volume.id,
'size': volume_size})
info_path = self._local_path_volume_info(snapshot.volume)
snap_info = self._read_info_file(info_path)
vol_path = self._local_volume_dir(snapshot.volume)
forward_file = snap_info[snapshot.id]
forward_path = os.path.join(vol_path, forward_file)
self._ensure_shares_mounted()
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path,
snapshot.volume.name)
path_to_snap_img = os.path.join(vol_path, img_info.backing_file)
path_to_new_vol = self._local_path_volume(volume)
path_to_cached_vol = self._local_volume_from_snap_cache_path(snapshot)
LOG.debug("will copy from snapshot at %s", path_to_snap_img)
if self.configuration.quobyte_qcow2_volumes:
out_format = 'qcow2'
else:
out_format = 'raw'
if not self.configuration.quobyte_volume_from_snapshot_cache:
LOG.debug("Creating direct copy from snapshot")
image_utils.convert_image(path_to_snap_img,
path_to_new_vol,
out_format,
run_as_root=self._execute_as_root)
else:
# create the volume via volume cache
if not os.access(path_to_cached_vol, os.F_OK):
LOG.debug("Caching volume %(volpath)s from snapshot.",
{'volpath': path_to_cached_vol})
image_utils.convert_image(path_to_snap_img,
path_to_cached_vol,
out_format,
run_as_root=self._execute_as_root)
# Copy volume from cache
LOG.debug("Copying volume %(volpath)s from cache",
{'volpath': path_to_new_vol})
shutil.copyfile(path_to_cached_vol, path_to_new_vol)
self._set_rw_permissions(path_to_new_vol)
self.optimize_volume(volume)
def copy_image_to_volume(self, context, volume, image_service, image_id):
with image_utils.temporary_file() as tmp:
# (wenhao): we don't need to convert to raw for sheepdog.
image_utils.fetch_verify_image(context, image_service, image_id, tmp)
# remove the image created by import before this function.
# see volume/drivers/manager.py:_create_volume
self.client.delete(volume.name)
# convert and store into sheepdog
image_utils.convert_image(tmp, self.client.local_path(volume), "raw")
self.client.resize(volume.name, volume.size)
def copy_image_to_volume(self, context, volume, image_service, image_id):
with image_utils.temporary_file() as tmp:
# (wenhao): we don't need to convert to raw for sheepdog.
image_utils.fetch_verify_image(context, image_service,
image_id, tmp)
# remove the image created by import before this function.
# see volume/drivers/manager.py:_create_volume
self._delete(volume)
# convert and store into sheepdog
image_utils.convert_image(tmp, 'sheepdog:%s' % volume['name'],
'raw')
self._resize(volume)
def test_convert_image(self):
mox = self._mox
mox.StubOutWithMock(utils, "execute")
TEST_OUT_FORMAT = "vmdk"
TEST_SOURCE = "img/qemu.img"
TEST_DEST = "/img/vmware.vmdk"
utils.execute("qemu-img", "convert", "-O", TEST_OUT_FORMAT, TEST_SOURCE, TEST_DEST, run_as_root=True)
mox.ReplayAll()
image_utils.convert_image(TEST_SOURCE, TEST_DEST, TEST_OUT_FORMAT)
mox.VerifyAll()
def _clone_image(self, volume, image_location, image_id):
"""Attempt to create a volume by efficiently copying image to volume.
If both source and target are backed by gpfs storage and the source
image is in raw format move the image to create a volume using either
gpfs clone operation or with a file copy. If the image format is not
raw, convert it to raw at the volume path.
"""
# Check if GPFS is mounted
self._verify_gpfs_path_state(self.configuration.gpfs_mount_point_base)
cloneable_image, reason, image_path = self._is_cloneable(image_id)
if not cloneable_image:
LOG.debug('Image %(img)s not cloneable: %(reas)s' %
{'img': image_id, 'reas': reason})
return (None, False)
vol_path = self.local_path(volume)
# if the image is not already a GPFS snap file make it so
if not self._is_gpfs_parent_file(image_path):
self._create_gpfs_snap(image_path, modebits='666')
data = image_utils.qemu_img_info(image_path)
# if image format is already raw either clone it or
# copy it depending on config file settings
if data.file_format == 'raw':
if (self.configuration.gpfs_images_share_mode ==
'copy_on_write'):
LOG.debug('Clone image to vol %s using mmclone' %
volume['id'])
self._create_gpfs_copy(image_path, vol_path)
elif self.configuration.gpfs_images_share_mode == 'copy':
LOG.debug('Clone image to vol %s using copyfile' %
volume['id'])
shutil.copyfile(image_path, vol_path)
self._execute('chmod', '666', vol_path, run_as_root=True)
# if image is not raw convert it to raw into vol_path destination
else:
LOG.debug('Clone image to vol %s using qemu convert' %
volume['id'])
image_utils.convert_image(image_path, vol_path, 'raw')
self._execute('chmod', '666', vol_path, run_as_root=True)
self._resize_volume_file(volume, volume['size'])
return {'provider_location': None}, True
def _direct_clone(self, volume, image_location, image_id, image_name):
"""Clones directly in nfs share."""
LOG.info('Checking image clone %s from glance share.', image_id)
cloned = False
image_location = self._get_image_nfs_url(image_location)
share = self._is_cloneable_share(image_location)
run_as_root = self._execute_as_root
dst_share = None
for dst in self._mounted_shares:
if dst and self._is_share_vol_compatible(volume, dst):
dst_share = dst
LOG.debug('Image dst share: %s', dst)
break
if not dst_share:
return cloned
LOG.debug('Share is cloneable %s', dst_share)
volume['provider_location'] = dst_share
(__, ___, img_file) = image_location.rpartition('/')
dir_path = self._get_mount_point_for_share(share)
dst_path = self._get_mount_point_for_share(dst_share)
img_path = '%s/%s' % (dir_path, img_file)
img_info = image_utils.qemu_img_info(img_path,
run_as_root=run_as_root)
if img_info.file_format == 'raw':
LOG.debug('Image is raw %s', image_id)
self._clone_volume_to_volume(
img_file, volume['name'], image_name,
volume_id=None, share=share, dst=dst_share, image_id=image_id)
cloned = True
else:
LOG.info('Image will locally be converted to raw %s',
image_id)
dst = '%s/%s' % (dst_path, volume['name'])
image_utils.convert_image(img_path, dst, 'raw',
run_as_root=run_as_root)
data = image_utils.qemu_img_info(dst, run_as_root=run_as_root)
if data.file_format != "raw":
raise exception.InvalidResults(
_('Converted to raw, but '
'format is now %s') % data.file_format)
else:
cloned = True
self._create_image_snapshot(
volume['name'], volume['provider_location'],
image_id, image_name)
return cloned
def test_convert_image(self):
mox = self._mox
mox.StubOutWithMock(utils, 'execute')
TEST_OUT_FORMAT = 'vmdk'
TEST_SOURCE = 'img/qemu.img'
TEST_DEST = '/img/vmware.vmdk'
utils.execute('qemu-img', 'convert', '-O', TEST_OUT_FORMAT,
TEST_SOURCE, TEST_DEST, run_as_root=True)
mox.ReplayAll()
image_utils.convert_image(TEST_SOURCE, TEST_DEST, TEST_OUT_FORMAT)
mox.VerifyAll()
def copy_image_to_volume(self, context, volume, image_service, image_id):
# use the image_conversion_dir as a temporary place to save the image
conversion_dir = CONF.image_conversion_dir
self._ensure_dir_exists(conversion_dir)
with tempfile.NamedTemporaryFile(dir=conversion_dir) as tmp:
# (wenhao): we don't need to convert to raw for sheepdog.
image_utils.fetch_verify_image(context, image_service,
image_id, tmp.name)
# remove the image created by import before this function.
# see volume/drivers/manager.py:_create_volume
self._delete(volume)
# convert and store into sheepdog
image_utils.convert_image(tmp, 'sheepdog:%s' % volume['name'],
'raw')
self._resize(volume)
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2.
"""
info_path = self._local_path_volume_info(snapshot.volume)
snap_info = self._read_info_file(info_path)
vol_dir = self._local_volume_dir(snapshot.volume)
out_format = self.choose_volume_format(volume)
volume_format = self.get_volume_format(snapshot.volume)
volume_path = self.local_path(volume)
if volume_format in (DISK_FORMAT_QCOW2, DISK_FORMAT_RAW):
forward_file = snap_info[snapshot.id]
forward_path = os.path.join(vol_dir, forward_file)
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path,
snapshot.volume.name)
path_to_snap_img = os.path.join(vol_dir, img_info.backing_file)
LOG.debug("_copy_volume_from_snapshot: will copy "
"from snapshot at %s.", path_to_snap_img)
image_utils.convert_image(path_to_snap_img,
volume_path,
out_format)
elif volume_format == DISK_FORMAT_PLOOP:
with PloopDevice(self.local_path(snapshot.volume),
snapshot.id,
execute=self._execute) as dev:
image_utils.convert_image(dev, volume_path, out_format)
else:
msg = _("Unsupported volume format %s") % volume_format
raise exception.InvalidVolume(msg)
if out_format == DISK_FORMAT_PLOOP:
img_path = os.path.join(volume_path, 'root.hds')
os.rename(volume_path, volume_path + '.tmp')
os.mkdir(volume_path)
os.rename(volume_path + '.tmp', img_path)
self._execute('ploop', 'restore-descriptor', volume_path, img_path)
self._extend_volume(volume, volume_size, out_format)
def test_clone_image_resizefails(self):
drv = self._driver
mox = self.mox
volume = {'name': 'vol', 'size': '20'}
mox.StubOutWithMock(drv, '_find_image_in_cache')
mox.StubOutWithMock(drv, '_is_cloneable_share')
mox.StubOutWithMock(drv, '_get_mount_point_for_share')
mox.StubOutWithMock(image_utils, 'qemu_img_info')
mox.StubOutWithMock(drv, '_clone_volume')
mox.StubOutWithMock(drv, '_discover_file_till_timeout')
mox.StubOutWithMock(drv, '_set_rw_permissions_for_all')
mox.StubOutWithMock(drv, '_resize_image_file')
mox.StubOutWithMock(image_utils, 'convert_image')
mox.StubOutWithMock(drv, '_register_image_in_cache')
mox.StubOutWithMock(drv, '_is_share_vol_compatible')
mox.StubOutWithMock(drv, 'local_path')
mox.StubOutWithMock(os.path, 'exists')
mox.StubOutWithMock(drv, '_delete_file')
drv._find_image_in_cache(IgnoreArg()).AndReturn([])
drv._is_cloneable_share('nfs://127.0.0.1/share/img-id').AndReturn(
'127.0.0.1:/share')
drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True)
drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt')
image_utils.qemu_img_info('/mnt/img-id').AndReturn(
self.get_img_info('notraw'))
image_utils.convert_image(IgnoreArg(), IgnoreArg(), 'raw')
image_utils.qemu_img_info('/mnt/vol').AndReturn(
self.get_img_info('raw'))
drv._register_image_in_cache(IgnoreArg(), IgnoreArg())
drv.local_path(IgnoreArg()).AndReturn('/mnt/vol')
drv._discover_file_till_timeout(IgnoreArg()).AndReturn(True)
drv._set_rw_permissions_for_all('/mnt/vol')
drv._resize_image_file(
IgnoreArg(), IgnoreArg()).AndRaise(exception.InvalidResults())
drv.local_path(IgnoreArg()).AndReturn('/mnt/vol')
os.path.exists('/mnt/vol').AndReturn(True)
drv._delete_file('/mnt/vol')
mox.ReplayAll()
vol_dict, result = drv. clone_image(
volume, ('nfs://127.0.0.1/share/img-id', None), 'image_id', {})
mox.VerifyAll()
self.assertFalse(result)
self.assertFalse(vol_dict['bootable'])
self.assertIsNone(vol_dict['provider_location'])
def _gpfs_fetch_to_raw(self, context, image_service, image_id, dest,
user_id=None, project_id=None):
if (self.configuration.image_conversion_dir and not
os.path.exists(self.configuration.image_conversion_dir)):
os.makedirs(self.configuration.image_conversion_dir)
tmp = "%s.part" % dest
with fileutils.remove_path_on_error(tmp):
self._gpfs_fetch(context, image_service, image_id, tmp, user_id,
project_id)
data = image_utils.qemu_img_info(tmp)
fmt = data.file_format
backing_file = data.backing_file
if backing_file is not None:
msg = (_("fmt = %(fmt)s backed by: %(backing_file)s") %
{'fmt': fmt, 'backing_file': backing_file})
LOG.error(msg)
raise exception.ImageUnacceptable(
image_id=image_id,
reason=msg)
if fmt is None:
msg = _("'qemu-img info' parsing failed.")
LOG.error(msg)
raise exception.ImageUnacceptable(
reason=msg,
image_id=image_id)
elif fmt == 'raw': # already in raw format - just rename to dest
self._execute('mv', tmp, dest, run_as_root=True)
else: # conversion to raw format required
LOG.debug("%s was %s, converting to raw" % (image_id, fmt))
image_utils.convert_image(tmp, dest, 'raw')
os.unlink(tmp)
data = image_utils.qemu_img_info(dest)
if data.file_format != "raw":
msg = (_("Expected image to be in raw format, but is %s") %
data.file_format)
LOG.error(msg)
raise exception.ImageUnacceptable(
image_id=image_id,
reason=msg)
return {'size': math.ceil(data.virtual_size / 1024.0 ** 3)}
def _do_extend_volume(self, volume_path, size_gb, volume_name):
info = self._qemu_img_info(volume_path, volume_name)
fmt = info.file_format
# Note(lpetrut): as for version 2.0, qemu-img cannot resize
# vhd/x images. For the moment, we'll just use an intermediary
# conversion in order to be able to do the resize.
if fmt in (self._DISK_FORMAT_VHDX, self._DISK_FORMAT_VHD_LEGACY):
temp_image = volume_path + ".tmp"
image_utils.convert_image(volume_path, temp_image, self._DISK_FORMAT_RAW)
image_utils.resize_image(temp_image, size_gb)
image_utils.convert_image(temp_image, volume_path, fmt)
self._delete(temp_image)
else:
image_utils.resize_image(volume_path, size_gb)
if not self._is_file_size_equal(volume_path, size_gb):
raise exception.ExtendVolumeError(reason="Resizing image file failed.")
def copy_image_to_volume(self, context, volume, image_service, image_id):
with image_utils.temporary_file() as tmp:
# (wenhao): we don't need to convert to raw for sheepdog.
image_utils.fetch_verify_image(context, image_service,
image_id, tmp)
# remove the image created by import before this function.
# see volume/drivers/manager.py:_create_volume
self.client.delete(volume.name)
# convert and store into sheepdog
image_utils.convert_image(
tmp,
'sheepdog:%(addr)s:%(port)d:%(name)s' % {
'addr': CONF.sheepdog_store_address,
'port': CONF.sheepdog_store_port,
'name': volume['name']},
'raw')
self.client.resize(volume.name, volume.size)
def _clone_image(self, volume, image_location, image_id):
"""Attempt to create a volume by efficiently copying image to volume.
If both source and target are backed by gpfs storage and the source
image is in raw format move the image to create a volume using either
gpfs clone operation or with a file copy. If the image format is not
raw, convert it to raw at the volume path.
"""
# Check if GPFS is mounted
self._verify_gpfs_path_state(self.configuration.gpfs_mount_point_base)
cloneable_image, reason, image_path = self._is_cloneable(image_id)
if not cloneable_image:
LOG.debug("Image %(img)s not cloneable: %(reas)s." % {"img": image_id, "reas": reason})
return (None, False)
vol_path = self.local_path(volume)
data = image_utils.qemu_img_info(image_path)
# if image format is already raw either clone it or
# copy it depending on config file settings
if data.file_format == "raw":
if self.configuration.gpfs_images_share_mode == "copy_on_write":
LOG.debug("Clone image to vol %s using mmclone." % volume["id"])
# if the image is not already a GPFS snap file make it so
if not self._is_gpfs_parent_file(image_path):
self._create_gpfs_snap(image_path)
self._create_gpfs_copy(image_path, vol_path)
elif self.configuration.gpfs_images_share_mode == "copy":
LOG.debug("Clone image to vol %s using copyfile." % volume["id"])
shutil.copyfile(image_path, vol_path)
# if image is not raw convert it to raw into vol_path destination
else:
LOG.debug("Clone image to vol %s using qemu convert." % volume["id"])
image_utils.convert_image(image_path, vol_path, "raw")
self._set_rw_permission(vol_path)
self._resize_volume_file(volume, volume["size"])
return {"provider_location": None}, True
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2.
"""
info_path = self._local_path_volume_info(snapshot.volume)
snap_info = self._read_info_file(info_path)
vol_dir = self._local_volume_dir(snapshot.volume)
out_format = self.choose_volume_format(volume)
qemu_out_format = image_utils.fixup_disk_format(out_format)
volume_format = self.get_volume_format(snapshot.volume)
volume_path = self.local_path(volume)
if volume_format in (DISK_FORMAT_QCOW2, DISK_FORMAT_RAW):
forward_file = snap_info[snapshot.id]
forward_path = os.path.join(vol_dir, forward_file)
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path, snapshot.volume.name)
path_to_snap_img = os.path.join(vol_dir, img_info.backing_file)
LOG.debug(
"_copy_volume_from_snapshot: will copy "
"from snapshot at %s.", path_to_snap_img)
image_utils.convert_image(path_to_snap_img, volume_path,
qemu_out_format)
elif volume_format == DISK_FORMAT_PLOOP:
with PloopDevice(self.local_path(snapshot.volume),
snapshot.id,
execute=self._execute) as dev:
base_file = os.path.join(volume_path, 'root.hds')
image_utils.convert_image(dev, base_file, qemu_out_format)
else:
msg = _("Unsupported volume format %s") % volume_format
raise exception.InvalidVolume(msg)
self._extend_volume(volume, volume_size, out_format)
# Query qemu-img info to cache the output
img_info = self._qemu_img_info(volume_path, volume.name)
def _direct_nfs_clone(self, volume, image_location, image_id):
"""Clone directly in nfs share."""
LOG.info(_LI('Checking image clone %s from glance share.'), image_id)
cloned = False
image_locations = self._construct_image_nfs_url(image_location)
run_as_root = self._execute_as_root
for loc in image_locations:
share = self._is_cloneable_share(loc)
if share and self._is_share_vol_compatible(volume, share):
LOG.debug('Share is cloneable %s', share)
volume['provider_location'] = share
(__, ___, img_file) = loc.rpartition('/')
dir_path = self._get_mount_point_for_share(share)
img_path = '%s/%s' % (dir_path, img_file)
img_info = image_utils.qemu_img_info(img_path,
run_as_root=run_as_root)
if img_info.file_format == 'raw':
LOG.debug('Image is raw %s', image_id)
self._clone_backing_file_for_volume(img_file,
volume['name'],
volume_id=None,
share=share)
cloned = True
break
else:
LOG.info(_LI('Image will locally be converted to raw %s'),
image_id)
dst = '%s/%s' % (dir_path, volume['name'])
image_utils.convert_image(img_path,
dst,
'raw',
run_as_root=run_as_root)
data = image_utils.qemu_img_info(dst,
run_as_root=run_as_root)
if data.file_format != "raw":
raise exception.InvalidResults(
_("Converted to raw, but"
" format is now %s") % data.file_format)
else:
cloned = True
self._register_image_in_cache(volume, image_id)
break
return cloned
def _direct_nfs_clone(self, volume, image_location, image_id):
"""Clone directly in nfs share."""
LOG.info(_LI('Checking image clone %s from glance share.'), image_id)
cloned = False
image_locations = self._construct_image_nfs_url(image_location)
run_as_root = self._execute_as_root
for loc in image_locations:
share = self._is_cloneable_share(loc)
if share and self._is_share_vol_compatible(volume, share):
LOG.debug('Share is cloneable %s', share)
volume['provider_location'] = share
(__, ___, img_file) = loc.rpartition('/')
dir_path = self._get_mount_point_for_share(share)
img_path = '%s/%s' % (dir_path, img_file)
img_info = image_utils.qemu_img_info(img_path,
run_as_root=run_as_root)
if img_info.file_format == 'raw':
LOG.debug('Image is raw %s', image_id)
self._clone_backing_file_for_volume(
img_file, volume['name'],
volume_id=None, share=share)
cloned = True
break
else:
LOG.info(
_LI('Image will locally be converted to raw %s'),
image_id)
dst = '%s/%s' % (dir_path, volume['name'])
image_utils.convert_image(img_path, dst, 'raw',
run_as_root=run_as_root)
data = image_utils.qemu_img_info(dst,
run_as_root=run_as_root)
if data.file_format != "raw":
raise exception.InvalidResults(
_("Converted to raw, but"
" format is now %s") % data.file_format)
else:
cloned = True
self._register_image_in_cache(
volume, image_id)
break
return cloned
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2.
"""
info_path = self._local_path_volume_info(snapshot['volume'])
# For BC compat' with version < 2 of this driver
try:
snap_info = self._read_info_file(info_path)
except IOError as exc:
if exc.errno != errno.ENOENT:
raise
else:
path_to_snap_img = self.local_path(snapshot)
else:
vol_path = self._local_volume_dir(snapshot['volume'])
forward_file = snap_info[snapshot['id']]
forward_path = os.path.join(vol_path, forward_file)
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path,
snapshot['volume']['name'])
path_to_snap_img = os.path.join(vol_path, img_info.backing_file)
LOG.debug("will copy from snapshot at %s", path_to_snap_img)
path_to_new_vol = self.local_path(volume)
out_format = 'raw'
image_utils.convert_image(path_to_snap_img,
path_to_new_vol,
out_format,
run_as_root=self._execute_as_root)
self._set_rw_permissions_for_all(path_to_new_vol)
image_utils.resize_image(path_to_new_vol, volume_size)
def _do_extend_volume(self, volume_path, size_gb, volume_name):
info = self._qemu_img_info(volume_path, volume_name)
fmt = info.file_format
# Note(lpetrut): as for version 2.0, qemu-img cannot resize
# vhd/x images. For the moment, we'll just use an intermediary
# conversion in order to be able to do the resize.
if fmt in (self._DISK_FORMAT_VHDX, self._DISK_FORMAT_VHD_LEGACY):
temp_image = volume_path + '.tmp'
image_utils.convert_image(volume_path, temp_image,
self._DISK_FORMAT_RAW)
image_utils.resize_image(temp_image, size_gb)
image_utils.convert_image(temp_image, volume_path, fmt)
self._delete(temp_image)
else:
image_utils.resize_image(volume_path, size_gb)
if not self._is_file_size_equal(volume_path, size_gb):
raise exception.ExtendVolumeError(
reason='Resizing image file failed.')
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
# If snapshots exist, flatten to a temporary image, and upload it
active_file = self.get_active_image_from_info(volume)
active_file_path = '%s/%s' % (self._local_volume_dir(volume),
active_file)
info = self._qemu_img_info(active_file_path, volume['name'])
backing_file = info.backing_file
if backing_file:
snapshots_exist = True
else:
snapshots_exist = False
root_file_fmt = info.file_format
temp_path = None
try:
if snapshots_exist or (root_file_fmt != 'raw'):
# Convert due to snapshots
# or volume data not being stored in raw format
# (upload_volume assumes raw format input)
temp_path = '%s/%s.temp_image.%s' % (
self._local_volume_dir(volume),
volume['id'],
image_meta['id'])
image_utils.convert_image(active_file_path, temp_path, 'raw')
upload_path = temp_path
else:
upload_path = active_file_path
image_utils.upload_volume(context,
image_service,
image_meta,
upload_path)
finally:
if temp_path is not None:
self._execute('rm', '-f', temp_path)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
# If snapshots exist, flatten to a temporary image, and upload it
active_file = self.get_active_image_from_info(volume)
active_file_path = '%s/%s' % (self._local_volume_dir(volume),
active_file)
info = self._qemu_img_info(active_file_path)
backing_file = info.backing_file
if backing_file:
snapshots_exist = True
else:
snapshots_exist = False
root_file_fmt = info.file_format
temp_path = None
try:
if snapshots_exist or (root_file_fmt != 'raw'):
# Convert due to snapshots
# or volume data not being stored in raw format
# (upload_volume assumes raw format input)
temp_path = '%s/%s.temp_image.%s' % (
self._local_volume_dir(volume),
volume['id'],
image_meta['id'])
image_utils.convert_image(active_file_path, temp_path, 'raw')
upload_path = temp_path
else:
upload_path = active_file_path
image_utils.upload_volume(context,
image_service,
image_meta,
upload_path)
finally:
if temp_path is not None:
self._execute('rm', '-f', temp_path)
def test_clone_image_file_not_discovered(self):
drv = self._driver
mox = self.mox
volume = {'name': 'vol', 'size': '20'}
mox.StubOutWithMock(drv, '_find_image_in_cache')
mox.StubOutWithMock(drv, '_is_cloneable_share')
mox.StubOutWithMock(drv, '_get_mount_point_for_share')
mox.StubOutWithMock(image_utils, 'qemu_img_info')
mox.StubOutWithMock(drv, '_clone_volume')
mox.StubOutWithMock(drv, '_discover_file_till_timeout')
mox.StubOutWithMock(image_utils, 'convert_image')
mox.StubOutWithMock(drv, '_register_image_in_cache')
mox.StubOutWithMock(drv, '_is_share_vol_compatible')
mox.StubOutWithMock(drv, 'local_path')
mox.StubOutWithMock(os.path, 'exists')
mox.StubOutWithMock(drv, '_delete_file')
drv._find_image_in_cache(IgnoreArg()).AndReturn([])
drv._is_cloneable_share('nfs://127.0.0.1/share/img-id').AndReturn(
'127.0.0.1:/share')
drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True)
drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt')
image_utils.qemu_img_info('/mnt/img-id').AndReturn(
self.get_img_info('notraw'))
image_utils.convert_image(IgnoreArg(), IgnoreArg(), 'raw')
image_utils.qemu_img_info('/mnt/vol').AndReturn(
self.get_img_info('raw'))
drv._register_image_in_cache(IgnoreArg(), IgnoreArg())
drv.local_path(IgnoreArg()).AndReturn('/mnt/vol')
drv._discover_file_till_timeout(IgnoreArg()).AndReturn(False)
drv.local_path(IgnoreArg()).AndReturn('/mnt/vol')
os.path.exists('/mnt/vol').AndReturn(True)
drv._delete_file('/mnt/vol')
mox.ReplayAll()
vol_dict, result = drv.clone_image(
volume, ('nfs://127.0.0.1/share/img-id', None), 'image_id', {})
mox.VerifyAll()
self.assertFalse(result)
self.assertFalse(vol_dict['bootable'])
self.assertIsNone(vol_dict['provider_location'])
def test_clone_image_resizefails(self):
drv = self._driver
mox = self.mox
volume = {"name": "vol", "size": "20"}
mox.StubOutWithMock(drv, "_find_image_in_cache")
mox.StubOutWithMock(drv, "_is_cloneable_share")
mox.StubOutWithMock(drv, "_get_mount_point_for_share")
mox.StubOutWithMock(image_utils, "qemu_img_info")
mox.StubOutWithMock(drv, "_clone_volume")
mox.StubOutWithMock(drv, "_discover_file_till_timeout")
mox.StubOutWithMock(drv, "_set_rw_permissions_for_all")
mox.StubOutWithMock(drv, "_resize_image_file")
mox.StubOutWithMock(image_utils, "convert_image")
mox.StubOutWithMock(drv, "_register_image_in_cache")
mox.StubOutWithMock(drv, "_is_share_vol_compatible")
mox.StubOutWithMock(drv, "local_path")
mox.StubOutWithMock(os.path, "exists")
mox.StubOutWithMock(drv, "_delete_file")
drv._find_image_in_cache(IgnoreArg()).AndReturn([])
drv._is_cloneable_share("nfs://127.0.0.1/share/img-id").AndReturn("127.0.0.1:/share")
drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True)
drv._get_mount_point_for_share("127.0.0.1:/share").AndReturn("/mnt")
image_utils.qemu_img_info("/mnt/img-id").AndReturn(self.get_img_info("notraw"))
image_utils.convert_image(IgnoreArg(), IgnoreArg(), "raw")
image_utils.qemu_img_info("/mnt/vol").AndReturn(self.get_img_info("raw"))
drv._register_image_in_cache(IgnoreArg(), IgnoreArg())
drv.local_path(IgnoreArg()).AndReturn("/mnt/vol")
drv._discover_file_till_timeout(IgnoreArg()).AndReturn(True)
drv._set_rw_permissions_for_all("/mnt/vol")
drv._resize_image_file(IgnoreArg(), IgnoreArg()).AndRaise(exception.InvalidResults())
drv.local_path(IgnoreArg()).AndReturn("/mnt/vol")
os.path.exists("/mnt/vol").AndReturn(True)
drv._delete_file("/mnt/vol")
mox.ReplayAll()
vol_dict, result = drv.clone_image(volume, ("nfs://127.0.0.1/share/img-id", None), "image_id")
mox.VerifyAll()
self.assertFalse(result)
self.assertFalse(vol_dict["bootable"])
self.assertIsNone(vol_dict["provider_location"])
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2.
"""
LOG.debug("snapshot: %(snap)s, volume: %(vol)s, "
"volume_size: %(size)s" % {
'snap': snapshot['id'],
'vol': volume['id'],
'size': volume_size
})
info_path = self._local_path_volume_info(snapshot['volume'])
snap_info = self._read_info_file(info_path)
vol_path = self._local_volume_dir(snapshot['volume'])
forward_file = snap_info[snapshot['id']]
forward_path = os.path.join(vol_path, forward_file)
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path,
snapshot['volume']['name'])
path_to_snap_img = os.path.join(vol_path, img_info.backing_file)
path_to_new_vol = self._local_path_volume(volume)
LOG.debug("will copy from snapshot at %s" % path_to_snap_img)
if self.configuration.quobyte_qcow2_volumes:
out_format = 'qcow2'
else:
out_format = 'raw'
image_utils.convert_image(path_to_snap_img,
path_to_new_vol,
out_format,
run_as_root=self._execute_as_root)
self._set_rw_permissions_for_all(path_to_new_vol)
def test_convert_image(self):
mox = self._mox
mox.StubOutWithMock(utils, 'execute')
TEST_OUT_FORMAT = 'vmdk'
TEST_SOURCE = 'img/qemu.img'
TEST_DEST = '/img/vmware.vmdk'
utils.execute('qemu-img',
'convert',
'-O',
TEST_OUT_FORMAT,
TEST_SOURCE,
TEST_DEST,
run_as_root=True)
mox.ReplayAll()
image_utils.convert_image(TEST_SOURCE, TEST_DEST, TEST_OUT_FORMAT)
mox.VerifyAll()
def test_convert_image(self, mock_stat):
mox = self._mox
mox.StubOutWithMock(utils, "execute")
mox.StubOutWithMock(utils, "is_blk_device")
TEST_OUT_FORMAT = "vmdk"
TEST_SOURCE = "img/qemu.img"
TEST_DEST = "/img/vmware.vmdk"
utils.is_blk_device(TEST_DEST).AndReturn(True)
utils.execute("dd", "count=0", "if=img/qemu.img", "of=/img/vmware.vmdk", "oflag=direct", run_as_root=True)
utils.execute(
"qemu-img", "convert", "-t", "none", "-O", TEST_OUT_FORMAT, TEST_SOURCE, TEST_DEST, run_as_root=True
)
mox.ReplayAll()
image_utils.convert_image(TEST_SOURCE, TEST_DEST, TEST_OUT_FORMAT, run_as_root=True)
mox.VerifyAll()
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2.
"""
LOG.debug("snapshot: %(snap)s, volume: %(vol)s, "
"volume_size: %(size)s"
% {'snap': snapshot['id'],
'vol': volume['id'],
'size': volume_size})
info_path = self._local_path_volume_info(snapshot['volume'])
snap_info = self._read_info_file(info_path)
vol_path = self._local_volume_dir(snapshot['volume'])
forward_file = snap_info[snapshot['id']]
forward_path = os.path.join(vol_path, forward_file)
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path,
snapshot['volume']['name'])
path_to_snap_img = os.path.join(vol_path, img_info.backing_file)
path_to_new_vol = self._local_path_volume(volume)
LOG.debug("will copy from snapshot at %s" % path_to_snap_img)
if self.configuration.glusterfs_qcow2_volumes:
out_format = 'qcow2'
else:
out_format = 'raw'
image_utils.convert_image(path_to_snap_img,
path_to_new_vol,
out_format)
self._set_rw_permissions_for_all(path_to_new_vol)
def copy_image_to_volume(self, context, volume, image_service, image_id):
"""Fetch the image from image_service and write it to the volume."""
img_cache_volume = {'name': 'img-%s' % image_id}
img_cache_snapshot = {'volume_name': img_cache_volume['name'],
'name': 'snap'}
if self._list_snapshots(volume):
# If the volume has snapshots, there is no efficient way to replace
# the contents. Do things the inefficient way.
image_utils.fetch_to_raw(context, image_service, image_id,
self.local_path(volume),
self.configuration.volume_dd_blocksize,
size=volume['size'])
else:
if not self.snapshot_exists(img_cache_snapshot):
with image_utils.temporary_file() as tmp:
image_utils.fetch_verify_image(context, image_service,
image_id, tmp)
qemu_info = image_utils.qemu_img_info(tmp)
virtual_size_gb = math.ceil(
qemu_info.virtual_size / (1024.0 ** 3))
img_cache_volume['size'] = virtual_size_gb
self.create_volume(img_cache_volume)
image_utils.convert_image(
tmp, self.local_path(img_cache_volume), 'raw', None,
sparse=64 * 1024)
self.create_snapshot(img_cache_snapshot)
self.delete_volume(volume)
self.create_volume_from_snapshot(volume, img_cache_snapshot)
self.extend_volume(volume, volume['size'])
def test_convert_image(self, mock_stat):
mox = self._mox
mox.StubOutWithMock(utils, 'execute')
mox.StubOutWithMock(utils, 'is_blk_device')
TEST_OUT_FORMAT = 'vmdk'
TEST_SOURCE = 'img/qemu.img'
TEST_DEST = '/img/vmware.vmdk'
utils.is_blk_device(TEST_DEST).AndReturn(True)
utils.execute('dd', 'count=0', 'if=img/qemu.img',
'of=/img/vmware.vmdk', 'oflag=direct',
run_as_root=True)
utils.execute(
'qemu-img', 'convert', '-t', 'none', '-O', TEST_OUT_FORMAT,
TEST_SOURCE, TEST_DEST, run_as_root=True)
mox.ReplayAll()
image_utils.convert_image(TEST_SOURCE, TEST_DEST, TEST_OUT_FORMAT)
mox.VerifyAll()
def test_clone_image_cloneableshare_notraw(self):
drv = self._driver
mox = self.mox
volume = {'name': 'vol', 'size': '20'}
mox.StubOutWithMock(drv, '_find_image_in_cache')
mox.StubOutWithMock(drv, '_is_cloneable_share')
mox.StubOutWithMock(drv, '_get_mount_point_for_share')
mox.StubOutWithMock(image_utils, 'qemu_img_info')
mox.StubOutWithMock(drv, '_clone_volume')
mox.StubOutWithMock(drv, '_discover_file_till_timeout')
mox.StubOutWithMock(drv, '_set_rw_permissions_for_all')
mox.StubOutWithMock(drv, '_resize_image_file')
mox.StubOutWithMock(image_utils, 'convert_image')
mox.StubOutWithMock(drv, '_register_image_in_cache')
mox.StubOutWithMock(drv, '_is_share_vol_compatible')
drv._find_image_in_cache(IgnoreArg()).AndReturn([])
drv._is_cloneable_share('nfs://127.0.0.1/share/img-id').AndReturn(
'127.0.0.1:/share')
drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True)
drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt')
image_utils.qemu_img_info('/mnt/img-id').AndReturn(
self.get_img_info('notraw'))
image_utils.convert_image(IgnoreArg(), IgnoreArg(), 'raw')
image_utils.qemu_img_info('/mnt/vol').AndReturn(
self.get_img_info('raw'))
drv._register_image_in_cache(IgnoreArg(), IgnoreArg())
drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt')
drv._discover_file_till_timeout(IgnoreArg()).AndReturn(True)
drv._set_rw_permissions_for_all('/mnt/vol')
drv._resize_image_file({'name': 'vol'}, IgnoreArg())
mox.ReplayAll()
drv.clone_image(volume, ('nfs://127.0.0.1/share/img-id', None),
'image_id', {})
mox.VerifyAll()
def _copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
# If snapshots exist, flatten to a temporary image, and upload it
active_file = self.get_active_image_from_info(volume)
active_file_path = os.path.join(self._local_volume_dir(volume), active_file)
info = self._qemu_img_info(active_file_path, volume["name"])
backing_file = info.backing_file
root_file_fmt = info.file_format
tmp_params = {"prefix": "%s.temp_image.%s" % (volume["id"], image_meta["id"]), "suffix": ".img"}
with image_utils.temporary_file(**tmp_params) as temp_path:
if backing_file or (root_file_fmt != "raw"):
# Convert due to snapshots
# or volume data not being stored in raw format
# (upload_volume assumes raw format input)
image_utils.convert_image(active_file_path, temp_path, "raw")
upload_path = temp_path
else:
upload_path = active_file_path
image_utils.upload_volume(context, image_service, image_meta, upload_path)
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2.
"""
LOG.debug(
"Snapshot: %(snap)s, volume: %(vol)s, "
"volume_size: %(size)s", {
'snap': snapshot['id'],
'vol': volume['id'],
'size': volume_size
})
info_path = self._local_path_volume_info(snapshot['volume'])
snap_info = self._read_info_file(info_path)
vol_dir = self._local_volume_dir(snapshot['volume'])
out_format = self.get_volume_format(volume, qemu_format=True)
forward_file = snap_info[snapshot['id']]
forward_path = os.path.join(vol_dir, forward_file)
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path,
snapshot['volume']['name'])
path_to_snap_img = os.path.join(vol_dir, img_info.backing_file)
LOG.debug("Will copy from snapshot at %s", path_to_snap_img)
image_utils.convert_image(path_to_snap_img, self.local_path(volume),
out_format)
self._extend_volume(volume, volume_size)
self._set_rw_permissions_for_all(self.local_path(volume))
def test_clone_image_cloneableshare_notraw(self):
drv = self._driver
mox = self.mox
volume = {'name': 'vol', 'size': '20'}
mox.StubOutWithMock(drv, '_find_image_in_cache')
mox.StubOutWithMock(drv, '_is_cloneable_share')
mox.StubOutWithMock(drv, '_get_mount_point_for_share')
mox.StubOutWithMock(image_utils, 'qemu_img_info')
mox.StubOutWithMock(drv, '_clone_volume')
mox.StubOutWithMock(drv, '_discover_file_till_timeout')
mox.StubOutWithMock(drv, '_set_rw_permissions_for_all')
mox.StubOutWithMock(drv, '_resize_image_file')
mox.StubOutWithMock(image_utils, 'convert_image')
mox.StubOutWithMock(drv, '_register_image_in_cache')
mox.StubOutWithMock(drv, '_is_share_vol_compatible')
drv._find_image_in_cache(IgnoreArg()).AndReturn([])
drv._is_cloneable_share('nfs://127.0.0.1/share/img-id').AndReturn(
'127.0.0.1:/share')
drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True)
drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt')
image_utils.qemu_img_info('/mnt/img-id').AndReturn(
self.get_img_info('notraw'))
image_utils.convert_image(IgnoreArg(), IgnoreArg(), 'raw')
image_utils.qemu_img_info('/mnt/vol').AndReturn(
self.get_img_info('raw'))
drv._register_image_in_cache(IgnoreArg(), IgnoreArg())
drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt')
drv._discover_file_till_timeout(IgnoreArg()).AndReturn(True)
drv._set_rw_permissions_for_all('/mnt/vol')
drv._resize_image_file({'name': 'vol'}, IgnoreArg())
mox.ReplayAll()
drv. clone_image(
volume, ('nfs://127.0.0.1/share/img-id', None), 'image_id', {})
mox.VerifyAll()
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size,
src_encryption_key_id=None,
new_encryption_key_id=None):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2.
"""
LOG.debug("Copying snapshot: %(snap)s -> volume: %(vol)s, "
"volume_size: %(size)s GB",
{'snap': snapshot.id,
'vol': volume.id,
'size': volume_size})
info_path = self._local_path_volume_info(snapshot.volume)
snap_info = self._read_info_file(info_path)
vol_path = self._local_volume_dir(snapshot.volume)
forward_file = snap_info[snapshot.id]
forward_path = os.path.join(vol_path, forward_file)
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path, snapshot.volume.name)
path_to_snap_img = os.path.join(vol_path, img_info.backing_file)
path_to_new_vol = self._local_path_volume(volume)
LOG.debug("will copy from snapshot at %s", path_to_snap_img)
if self.configuration.nfs_qcow2_volumes:
out_format = 'qcow2'
else:
out_format = 'raw'
if new_encryption_key_id is not None:
if src_encryption_key_id is None:
message = _("Can't create an encrypted volume %(format)s "
"from an unencrypted source."
) % {'format': out_format}
LOG.error(message)
# TODO(enriquetaso): handle unencrypted snap->encrypted vol
raise exception.NfsException(message)
keymgr = key_manager.API(CONF)
new_key = keymgr.get(volume.obj_context, new_encryption_key_id)
new_passphrase = \
binascii.hexlify(new_key.get_encoded()).decode('utf-8')
# volume.obj_context is the owner of this request
src_key = keymgr.get(volume.obj_context, src_encryption_key_id)
src_passphrase = \
binascii.hexlify(src_key.get_encoded()).decode('utf-8')
tmp_dir = volume_utils.image_conversion_dir()
with tempfile.NamedTemporaryFile(prefix='luks_',
dir=tmp_dir) as src_pass_file:
with open(src_pass_file.name, 'w') as f:
f.write(src_passphrase)
with tempfile.NamedTemporaryFile(prefix='luks_',
dir=tmp_dir) as new_pass_file:
with open(new_pass_file.name, 'w') as f:
f.write(new_passphrase)
image_utils.convert_image(
path_to_snap_img,
path_to_new_vol,
'luks',
passphrase_file=new_pass_file.name,
src_passphrase_file=src_pass_file.name,
run_as_root=self._execute_as_root)
else:
image_utils.convert_image(path_to_snap_img,
path_to_new_vol,
out_format,
run_as_root=self._execute_as_root)
self._set_rw_permissions_for_all(path_to_new_vol)
def _cache_vol_2_2(self, context, vol, image_meta, image_service):
image_id = image_meta['id']
# Pull down image and determine if valid
with image_utils.TemporaryImages.fetch(image_service, context,
image_id) as tmp_image:
data = image_utils.qemu_img_info(tmp_image)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(
reason=_("'qemu-img info' parsing failed."),
image_id=image_id)
backing_file = data.backing_file
if backing_file is not None:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {
'fmt': fmt,
'backing_file': backing_file,
})
vsize = int(math.ceil(float(data.virtual_size) / units.Gi))
vol['size'] = vsize
vtype = vol['volume_type_id']
LOG.info(
"Creating cached image with volume type: %(vtype)s and "
"size %(size)s", {
'vtype': vtype,
'size': vsize
})
self._create_volume_2_2(vol)
with self._connect_vol(context, vol) as device:
LOG.debug("Moving image %s to volume %s", image_meta['id'],
datc.get_name(vol))
image_utils.convert_image(tmp_image,
device,
'raw',
run_as_root=True)
LOG.debug("Finished moving image %s to volume %s",
image_meta['id'], datc.get_name(vol))
data = image_utils.qemu_img_info(device, run_as_root=True)
if data.file_format != 'raw':
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(vol_format)s, but format is "
"now %(file_format)s") % {
'vol_format': 'raw',
'file_format': data.file_format
})
# TODO(_alastor_): Remove this snapshot creation when we fix
# "created_at" attribute in the frontend
# We don't actually care about the snapshot uuid, we just want
# a single snapshot
snapshot = {
'id': str(uuid.uuid4()),
'volume_id': vol['id'],
'project_id': vol['project_id']
}
self._create_snapshot_2_2(snapshot)
metadata = {'type': 'cached_image'}
tenant = self.get_tenant(vol['project_id'])
ai = self.cvol_to_ai(vol, tenant=tenant)
ai.metadata.set(tenant=tenant, **metadata)
# Cloning offline AI is ~4 seconds faster than cloning online AI
self._detach_volume_2_2(None, vol)
def _copy_from_img_service(self, context, volume, image_service, image_id):
"""Copies from the image service using copy offload."""
LOG.debug("Trying copy from image service using copy offload.")
image_loc = image_service.get_location(context, image_id)
locations = self._construct_image_nfs_url(image_loc)
src_ip = None
selected_loc = None
cloned = False
# this will match the first location that has a valid IP on cluster
for location in locations:
conn, dr = self._check_get_nfs_path_segs(location)
if conn:
try:
src_ip = self._get_ip_verify_on_cluster(conn.split(':')[0])
selected_loc = location
break
except exception.NotFound:
pass
if src_ip is None:
raise exception.NotFound(_("Source host details not found."))
(__, ___, img_file) = selected_loc.rpartition('/')
src_path = os.path.join(dr, img_file)
dst_ip, vol_path = self._get_destination_ip_and_path(volume)
share_path = vol_path.rsplit("/", 1)[0]
dst_share = dst_ip + ':' + share_path
# tmp file is required to deal with img formats
tmp_img_file = six.text_type(uuid.uuid4())
col_path = self.configuration.netapp_copyoffload_tool_path
img_info = image_service.show(context, image_id)
self._check_share_can_hold_size(dst_share, img_info['size'])
run_as_root = self._execute_as_root
dst_dir = self._get_mount_point_for_share(dst_share)
dst_img_local = os.path.join(dst_dir, tmp_img_file)
try:
dst_img_serv_path = os.path.join(share_path, tmp_img_file)
# Always run copy offload as regular user, it's sufficient
# and rootwrap doesn't allow copy offload to run as root
# anyways.
self._execute(col_path,
src_ip,
dst_ip,
src_path,
dst_img_serv_path,
run_as_root=False,
check_exit_code=0)
self._discover_file_till_timeout(dst_img_local, timeout=120)
LOG.debug('Copied image %(img)s to tmp file %(tmp)s.', {
'img': image_id,
'tmp': tmp_img_file
})
dst_img_cache_local = os.path.join(dst_dir,
'img-cache-%s' % image_id)
if img_info['disk_format'] == 'raw':
LOG.debug('Image is raw %s.', image_id)
self._clone_file_dst_exists(dst_share,
tmp_img_file,
volume['name'],
dest_exists=True)
self._move_nfs_file(dst_img_local, dst_img_cache_local)
LOG.debug('Copied raw image %(img)s to volume %(vol)s.', {
'img': image_id,
'vol': volume['id']
})
else:
LOG.debug('Image will be converted to raw %s.', image_id)
img_conv = six.text_type(uuid.uuid4())
dst_img_conv_local = os.path.join(dst_dir, img_conv)
# Checking against image size which is approximate check
self._check_share_can_hold_size(dst_share, img_info['size'])
try:
image_utils.convert_image(dst_img_local,
dst_img_conv_local,
'raw',
run_as_root=run_as_root)
data = image_utils.qemu_img_info(dst_img_conv_local,
run_as_root=run_as_root)
if data.file_format != "raw":
raise exception.InvalidResults(
_("Converted to raw, but format is now %s.") %
data.file_format)
else:
self._clone_file_dst_exists(dst_share,
img_conv,
volume['name'],
dest_exists=True)
self._move_nfs_file(dst_img_conv_local,
dst_img_cache_local)
LOG.debug(
'Copied locally converted raw image'
' %(img)s to volume %(vol)s.', {
'img': image_id,
'vol': volume['id']
})
finally:
if os.path.exists(dst_img_conv_local):
self._delete_file_at_path(dst_img_conv_local)
cloned = True
finally:
if os.path.exists(dst_img_local):
self._delete_file_at_path(dst_img_local)
return cloned
def _copy_from_img_service(self, context, volume, image_service,
image_id):
"""Copies from the image service using copy offload."""
LOG.debug("Trying copy from image service using copy offload.")
image_loc = image_service.get_location(context, image_id)
image_loc = self._construct_image_nfs_url(image_loc)
conn, dr = self._check_get_nfs_path_segs(image_loc)
if conn:
src_ip = self._get_ip_verify_on_cluster(conn.split(':')[0])
else:
raise exception.NotFound(_("Source host details not found."))
(__, ___, img_file) = image_loc.rpartition('/')
src_path = os.path.join(dr, img_file)
dst_ip = self._get_ip_verify_on_cluster(self._get_host_ip(
volume['id']))
# tmp file is required to deal with img formats
tmp_img_file = six.text_type(uuid.uuid4())
col_path = self.configuration.netapp_copyoffload_tool_path
img_info = image_service.show(context, image_id)
dst_share = self._get_provider_location(volume['id'])
self._check_share_can_hold_size(dst_share, img_info['size'])
run_as_root = self._execute_as_root
dst_dir = self._get_mount_point_for_share(dst_share)
dst_img_local = os.path.join(dst_dir, tmp_img_file)
try:
# If src and dst share not equal
if (('%s:%s' % (src_ip, dr)) !=
('%s:%s' % (dst_ip, self._get_export_path(volume['id'])))):
dst_img_serv_path = os.path.join(
self._get_export_path(volume['id']), tmp_img_file)
self._execute(col_path, src_ip, dst_ip, src_path,
dst_img_serv_path, run_as_root=run_as_root,
check_exit_code=0)
else:
self._clone_file_dst_exists(dst_share, img_file, tmp_img_file)
self._discover_file_till_timeout(dst_img_local, timeout=120)
LOG.debug('Copied image %(img)s to tmp file %(tmp)s.',
{'img': image_id, 'tmp': tmp_img_file})
dst_img_cache_local = os.path.join(dst_dir,
'img-cache-%s' % image_id)
if img_info['disk_format'] == 'raw':
LOG.debug('Image is raw %s.', image_id)
self._clone_file_dst_exists(dst_share, tmp_img_file,
volume['name'], dest_exists=True)
self._move_nfs_file(dst_img_local, dst_img_cache_local)
LOG.debug('Copied raw image %(img)s to volume %(vol)s.',
{'img': image_id, 'vol': volume['id']})
else:
LOG.debug('Image will be converted to raw %s.', image_id)
img_conv = six.text_type(uuid.uuid4())
dst_img_conv_local = os.path.join(dst_dir, img_conv)
# Checking against image size which is approximate check
self._check_share_can_hold_size(dst_share, img_info['size'])
try:
image_utils.convert_image(dst_img_local,
dst_img_conv_local, 'raw',
run_as_root=run_as_root)
data = image_utils.qemu_img_info(dst_img_conv_local,
run_as_root=run_as_root)
if data.file_format != "raw":
raise exception.InvalidResults(
_("Converted to raw, but format is now %s.")
% data.file_format)
else:
self._clone_file_dst_exists(dst_share, img_conv,
volume['name'],
dest_exists=True)
self._move_nfs_file(dst_img_conv_local,
dst_img_cache_local)
LOG.debug('Copied locally converted raw image'
' %(img)s to volume %(vol)s.',
{'img': image_id, 'vol': volume['id']})
finally:
if os.path.exists(dst_img_conv_local):
self._delete_file_at_path(dst_img_conv_local)
self._post_clone_image(volume)
finally:
if os.path.exists(dst_img_local):
self._delete_file_at_path(dst_img_local)
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
"""Copy data from snapshot to destination volume.
This is done with a qemu-img convert to raw/qcow2 from the snapshot
qcow2. If the quobyte_volume_from_snapshot_cache is active the result
is written into the cache and all volumes created from this
snapshot id are created directly from the cache.
"""
LOG.debug("snapshot: %(snap)s, volume: %(vol)s, ", {
'snap': snapshot.id,
'vol': volume.id,
'size': volume_size
})
info_path = self._local_path_volume_info(snapshot.volume)
snap_info = self._read_info_file(info_path)
vol_path = self._local_volume_dir(snapshot.volume)
forward_file = snap_info[snapshot.id]
forward_path = os.path.join(vol_path, forward_file)
self._ensure_shares_mounted()
# Find the file which backs this file, which represents the point
# when this snapshot was created.
img_info = self._qemu_img_info(forward_path, snapshot.volume.name)
path_to_snap_img = os.path.join(vol_path, img_info.backing_file)
path_to_new_vol = self._local_path_volume(volume)
path_to_cached_vol = self._local_volume_from_snap_cache_path(snapshot)
LOG.debug("will copy from snapshot at %s", path_to_snap_img)
if self.configuration.quobyte_qcow2_volumes:
out_format = 'qcow2'
else:
out_format = 'raw'
if not self.configuration.quobyte_volume_from_snapshot_cache:
LOG.debug("Creating direct copy from snapshot")
image_utils.convert_image(path_to_snap_img,
path_to_new_vol,
out_format,
run_as_root=self._execute_as_root)
else:
# create the volume via volume cache
if not os.access(path_to_cached_vol, os.F_OK):
LOG.debug("Caching volume %(volpath)s from snapshot.",
{'volpath': path_to_cached_vol})
image_utils.convert_image(path_to_snap_img,
path_to_cached_vol,
out_format,
run_as_root=self._execute_as_root)
if self.configuration.quobyte_overlay_volumes:
# NOTE(kaisers): Create a parent symlink to track the
# existence of the parent
os.symlink(path_to_snap_img,
path_to_cached_vol + '.parent-' + snapshot.id)
if self.configuration.quobyte_overlay_volumes:
self._create_overlay_volume_from_snapshot(
volume, snapshot, volume_size, out_format)
else:
# Copy volume from cache
LOG.debug("Copying volume %(volpath)s from cache",
{'volpath': path_to_new_vol})
shutil.copyfile(path_to_cached_vol, path_to_new_vol)
# Note(kaisers): As writes beyond EOF are sequentialized with
# FUSE we call fallocate here to optimize performance:
self._fallocate_file(path_to_new_vol, volume_size)
self._set_rw_permissions(path_to_new_vol)
LOG.debug('Image is raw %s', image_id)
<<<<<<< HEAD
self._clone_backing_file_for_volume(
=======
self._clone_volume(
>>>>>>> refs/remotes/openstack/stable/kilo
img_file, volume['name'],
volume_id=None, share=share)
cloned = True
break
else:
LOG.info(
_LI('Image will locally be converted to raw %s'),
image_id)
dst = '%s/%s' % (dir_path, volume['name'])
image_utils.convert_image(img_path, dst, 'raw',
run_as_root=run_as_root)
data = image_utils.qemu_img_info(dst,
run_as_root=run_as_root)
if data.file_format != "raw":
raise exception.InvalidResults(
_("Converted to raw, but"
" format is now %s") % data.file_format)
else:
cloned = True
self._register_image_in_cache(
volume, image_id)
break
return cloned
def _post_clone_image(self, volume):
"""Do operations post image cloning."""
